| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9998854993048172, |
| "eval_steps": 100, |
| "global_step": 7641, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0, |
| "learning_rate": 6.535947712418301e-09, |
| "logits/chosen": -2.7937374114990234, |
| "logits/rejected": -2.696331262588501, |
| "logps/chosen": -219.9345245361328, |
| "logps/rejected": -238.54010009765625, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.0, |
| "learning_rate": 6.535947712418302e-08, |
| "logits/chosen": -2.686509370803833, |
| "logits/rejected": -2.617267370223999, |
| "logps/chosen": -272.2289123535156, |
| "logps/rejected": -247.8722381591797, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.2222222238779068, |
| "rewards/chosen": 0.00035665329778566957, |
| "rewards/margins": 0.0005560062127187848, |
| "rewards/rejected": -0.00019935290038120002, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.0, |
| "learning_rate": 1.3071895424836603e-07, |
| "logits/chosen": -2.6644845008850098, |
| "logits/rejected": -2.6715245246887207, |
| "logps/chosen": -264.3231201171875, |
| "logps/rejected": -269.95965576171875, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": 2.1979305529384874e-05, |
| "rewards/margins": -0.0005006279679946601, |
| "rewards/rejected": 0.0005226072971709073, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.0, |
| "learning_rate": 1.9607843137254904e-07, |
| "logits/chosen": -2.709402084350586, |
| "logits/rejected": -2.5456981658935547, |
| "logps/chosen": -235.95193481445312, |
| "logps/rejected": -211.9298095703125, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.4625000059604645, |
| "rewards/chosen": 9.268704889109358e-05, |
| "rewards/margins": -0.00016139161016326398, |
| "rewards/rejected": 0.0002540787390898913, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 2.6143790849673207e-07, |
| "logits/chosen": -2.7728488445281982, |
| "logits/rejected": -2.588787078857422, |
| "logps/chosen": -265.30828857421875, |
| "logps/rejected": -244.3712615966797, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.00027978085563518107, |
| "rewards/margins": -3.424427268328145e-05, |
| "rewards/rejected": -0.0002455365320201963, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 3.267973856209151e-07, |
| "logits/chosen": -2.763617515563965, |
| "logits/rejected": -2.7047462463378906, |
| "logps/chosen": -226.91055297851562, |
| "logps/rejected": -226.02536010742188, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -5.5276301281992346e-05, |
| "rewards/margins": -6.809332262491807e-05, |
| "rewards/rejected": 1.2817062270187307e-05, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 3.921568627450981e-07, |
| "logits/chosen": -2.7146997451782227, |
| "logits/rejected": -2.6671783924102783, |
| "logps/chosen": -267.3989562988281, |
| "logps/rejected": -241.76480102539062, |
| "loss": 0.6935, |
| "rewards/accuracies": 0.4124999940395355, |
| "rewards/chosen": -0.0004219438415020704, |
| "rewards/margins": -0.0006447834894061089, |
| "rewards/rejected": 0.00022283961880020797, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 4.5751633986928105e-07, |
| "logits/chosen": -2.7232730388641357, |
| "logits/rejected": -2.6747944355010986, |
| "logps/chosen": -245.4066619873047, |
| "logps/rejected": -204.679443359375, |
| "loss": 0.693, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.000368702836567536, |
| "rewards/margins": 0.0002510659396648407, |
| "rewards/rejected": 0.00011763688962673768, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 5.228758169934641e-07, |
| "logits/chosen": -2.68009614944458, |
| "logits/rejected": -2.608882427215576, |
| "logps/chosen": -289.00653076171875, |
| "logps/rejected": -281.58251953125, |
| "loss": 0.6927, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": 0.0005034831119701266, |
| "rewards/margins": 0.0008431966416537762, |
| "rewards/rejected": -0.00033971358789131045, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 5.882352941176471e-07, |
| "logits/chosen": -2.799229145050049, |
| "logits/rejected": -2.685251474380493, |
| "logps/chosen": -254.7516632080078, |
| "logps/rejected": -224.9294891357422, |
| "loss": 0.693, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.00010115172335645184, |
| "rewards/margins": 0.0002790264261420816, |
| "rewards/rejected": -0.000380178214982152, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 6.535947712418302e-07, |
| "logits/chosen": -2.6838011741638184, |
| "logits/rejected": -2.681504726409912, |
| "logps/chosen": -228.4124298095703, |
| "logps/rejected": -235.77685546875, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.0002754587621893734, |
| "rewards/margins": -0.0004044932429678738, |
| "rewards/rejected": 0.00012903442257083952, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.01, |
| "eval_logits/chosen": -2.526097059249878, |
| "eval_logits/rejected": -2.438264846801758, |
| "eval_logps/chosen": -268.4692077636719, |
| "eval_logps/rejected": -248.5731201171875, |
| "eval_loss": 0.6930866241455078, |
| "eval_rewards/accuracies": 0.5105000138282776, |
| "eval_rewards/chosen": 0.00020805322856176645, |
| "eval_rewards/margins": 0.00012786558363586664, |
| "eval_rewards/rejected": 8.018761582206935e-05, |
| "eval_runtime": 1145.3814, |
| "eval_samples_per_second": 1.746, |
| "eval_steps_per_second": 0.873, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 7.189542483660131e-07, |
| "logits/chosen": -2.694509506225586, |
| "logits/rejected": -2.6128382682800293, |
| "logps/chosen": -269.1925048828125, |
| "logps/rejected": -250.8614044189453, |
| "loss": 0.693, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": 0.0004071092698723078, |
| "rewards/margins": 0.00034786976175382733, |
| "rewards/rejected": 5.923947173869237e-05, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 7.843137254901962e-07, |
| "logits/chosen": -2.7306885719299316, |
| "logits/rejected": -2.6021299362182617, |
| "logps/chosen": -287.8159484863281, |
| "logps/rejected": -229.77554321289062, |
| "loss": 0.6927, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.0005465588765218854, |
| "rewards/margins": 0.0008666679495945573, |
| "rewards/rejected": -0.0003201091312803328, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 8.496732026143792e-07, |
| "logits/chosen": -2.693432569503784, |
| "logits/rejected": -2.6983115673065186, |
| "logps/chosen": -251.46865844726562, |
| "logps/rejected": -224.0875244140625, |
| "loss": 0.6926, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.0007204865687526762, |
| "rewards/margins": 0.001154972007498145, |
| "rewards/rejected": -0.0004344852641224861, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 9.150326797385621e-07, |
| "logits/chosen": -2.65818452835083, |
| "logits/rejected": -2.5886805057525635, |
| "logps/chosen": -259.5521545410156, |
| "logps/rejected": -231.7545928955078, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": 0.00015563381020911038, |
| "rewards/margins": -8.64746980369091e-05, |
| "rewards/rejected": 0.0002421085664536804, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 9.80392156862745e-07, |
| "logits/chosen": -2.792015314102173, |
| "logits/rejected": -2.6346828937530518, |
| "logps/chosen": -317.06451416015625, |
| "logps/rejected": -283.6150817871094, |
| "loss": 0.6927, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.000999806565232575, |
| "rewards/margins": 0.0009337253868579865, |
| "rewards/rejected": 6.60812875139527e-05, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 1.0457516339869283e-06, |
| "logits/chosen": -2.8084640502929688, |
| "logits/rejected": -2.663980007171631, |
| "logps/chosen": -247.31594848632812, |
| "logps/rejected": -224.99697875976562, |
| "loss": 0.6936, |
| "rewards/accuracies": 0.4000000059604645, |
| "rewards/chosen": -0.0001547201827634126, |
| "rewards/margins": -0.0008144931052811444, |
| "rewards/rejected": 0.0006597728352062404, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 1.111111111111111e-06, |
| "logits/chosen": -2.684959650039673, |
| "logits/rejected": -2.617687940597534, |
| "logps/chosen": -228.97103881835938, |
| "logps/rejected": -213.688232421875, |
| "loss": 0.693, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": 0.0012499226722866297, |
| "rewards/margins": 0.0002835305640473962, |
| "rewards/rejected": 0.0009663921082392335, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 1.1764705882352942e-06, |
| "logits/chosen": -2.8306772708892822, |
| "logits/rejected": -2.568779468536377, |
| "logps/chosen": -329.3360900878906, |
| "logps/rejected": -258.120361328125, |
| "loss": 0.6928, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": 0.0013785558985546231, |
| "rewards/margins": 0.0007068994455039501, |
| "rewards/rejected": 0.000671656453050673, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 1.2418300653594772e-06, |
| "logits/chosen": -2.6136114597320557, |
| "logits/rejected": -2.5621001720428467, |
| "logps/chosen": -249.232421875, |
| "logps/rejected": -210.09970092773438, |
| "loss": 0.6922, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.002042675856500864, |
| "rewards/margins": 0.0018692022422328591, |
| "rewards/rejected": 0.00017347374523524195, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.3071895424836604e-06, |
| "logits/chosen": -2.7648887634277344, |
| "logits/rejected": -2.6305103302001953, |
| "logps/chosen": -245.13864135742188, |
| "logps/rejected": -249.7960968017578, |
| "loss": 0.6924, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.0012468935456126928, |
| "rewards/margins": 0.0015361621044576168, |
| "rewards/rejected": -0.0002892684715334326, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.03, |
| "eval_logits/chosen": -2.5247368812561035, |
| "eval_logits/rejected": -2.436842918395996, |
| "eval_logps/chosen": -268.3450927734375, |
| "eval_logps/rejected": -248.5510711669922, |
| "eval_loss": 0.6925778388977051, |
| "eval_rewards/accuracies": 0.5605000257492065, |
| "eval_rewards/chosen": 0.0014493772760033607, |
| "eval_rewards/margins": 0.001148571027442813, |
| "eval_rewards/rejected": 0.000300806452287361, |
| "eval_runtime": 1160.8989, |
| "eval_samples_per_second": 1.723, |
| "eval_steps_per_second": 0.861, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.3725490196078434e-06, |
| "logits/chosen": -2.7637641429901123, |
| "logits/rejected": -2.705857515335083, |
| "logps/chosen": -267.3808898925781, |
| "logps/rejected": -244.5528564453125, |
| "loss": 0.6921, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.00269283726811409, |
| "rewards/margins": 0.0021984183695167303, |
| "rewards/rejected": 0.0004944190150126815, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.4379084967320261e-06, |
| "logits/chosen": -2.667719602584839, |
| "logits/rejected": -2.578129291534424, |
| "logps/chosen": -267.173095703125, |
| "logps/rejected": -245.6712646484375, |
| "loss": 0.692, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.0021157357841730118, |
| "rewards/margins": 0.0022508525289595127, |
| "rewards/rejected": -0.0001351167302345857, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.5032679738562091e-06, |
| "logits/chosen": -2.6373825073242188, |
| "logits/rejected": -2.600693941116333, |
| "logps/chosen": -218.6166534423828, |
| "logps/rejected": -262.50567626953125, |
| "loss": 0.6921, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": 0.0025270141195505857, |
| "rewards/margins": 0.0020475280471146107, |
| "rewards/rejected": 0.0004794862470589578, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.5686274509803923e-06, |
| "logits/chosen": -2.6723687648773193, |
| "logits/rejected": -2.6017861366271973, |
| "logps/chosen": -234.77505493164062, |
| "logps/rejected": -253.36123657226562, |
| "loss": 0.692, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.002189940307289362, |
| "rewards/margins": 0.0022566465195268393, |
| "rewards/rejected": -6.670653237961233e-05, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.6339869281045753e-06, |
| "logits/chosen": -2.7224369049072266, |
| "logits/rejected": -2.5702614784240723, |
| "logps/chosen": -301.28271484375, |
| "logps/rejected": -258.6485900878906, |
| "loss": 0.6917, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.003571895882487297, |
| "rewards/margins": 0.0029362873174250126, |
| "rewards/rejected": 0.0006356079829856753, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 1.6993464052287585e-06, |
| "logits/chosen": -2.75251841545105, |
| "logits/rejected": -2.6484122276306152, |
| "logps/chosen": -279.2475280761719, |
| "logps/rejected": -247.06002807617188, |
| "loss": 0.6915, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.004341104533523321, |
| "rewards/margins": 0.0033817340154200792, |
| "rewards/rejected": 0.0009593702852725983, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 1.7647058823529414e-06, |
| "logits/chosen": -2.6575112342834473, |
| "logits/rejected": -2.5978779792785645, |
| "logps/chosen": -235.12271118164062, |
| "logps/rejected": -213.41455078125, |
| "loss": 0.6907, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.004421571735292673, |
| "rewards/margins": 0.00482561532407999, |
| "rewards/rejected": -0.00040404353057965636, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 1.8300653594771242e-06, |
| "logits/chosen": -2.789543867111206, |
| "logits/rejected": -2.6499438285827637, |
| "logps/chosen": -304.09271240234375, |
| "logps/rejected": -272.1944274902344, |
| "loss": 0.6908, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.006846042815595865, |
| "rewards/margins": 0.004689323715865612, |
| "rewards/rejected": 0.0021567184012383223, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 1.8954248366013072e-06, |
| "logits/chosen": -2.6990113258361816, |
| "logits/rejected": -2.6313693523406982, |
| "logps/chosen": -274.0718688964844, |
| "logps/rejected": -256.3681335449219, |
| "loss": 0.6911, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.008100202307105064, |
| "rewards/margins": 0.0040775686502456665, |
| "rewards/rejected": 0.004022633656859398, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 1.96078431372549e-06, |
| "logits/chosen": -2.8378288745880127, |
| "logits/rejected": -2.6756560802459717, |
| "logps/chosen": -286.62457275390625, |
| "logps/rejected": -235.5787811279297, |
| "loss": 0.691, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.007958942092955112, |
| "rewards/margins": 0.004316599573940039, |
| "rewards/rejected": 0.00364234228618443, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.04, |
| "eval_logits/chosen": -2.5252809524536133, |
| "eval_logits/rejected": -2.4377505779266357, |
| "eval_logps/chosen": -267.5838928222656, |
| "eval_logps/rejected": -248.17530822753906, |
| "eval_loss": 0.6906724572181702, |
| "eval_rewards/accuracies": 0.6439999938011169, |
| "eval_rewards/chosen": 0.009060990996658802, |
| "eval_rewards/margins": 0.005002738442271948, |
| "eval_rewards/rejected": 0.004058253485709429, |
| "eval_runtime": 1152.1824, |
| "eval_samples_per_second": 1.736, |
| "eval_steps_per_second": 0.868, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 2.0261437908496734e-06, |
| "logits/chosen": -2.657287120819092, |
| "logits/rejected": -2.624457836151123, |
| "logps/chosen": -276.63470458984375, |
| "logps/rejected": -271.01629638671875, |
| "loss": 0.6908, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.008446435444056988, |
| "rewards/margins": 0.004816326312720776, |
| "rewards/rejected": 0.003630108432844281, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 2.0915032679738565e-06, |
| "logits/chosen": -2.745816707611084, |
| "logits/rejected": -2.5970892906188965, |
| "logps/chosen": -250.56851196289062, |
| "logps/rejected": -241.47689819335938, |
| "loss": 0.6909, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.009846082888543606, |
| "rewards/margins": 0.004527992103248835, |
| "rewards/rejected": 0.005318091716617346, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 2.1568627450980393e-06, |
| "logits/chosen": -2.8040456771850586, |
| "logits/rejected": -2.6397745609283447, |
| "logps/chosen": -284.6964416503906, |
| "logps/rejected": -238.743408203125, |
| "loss": 0.6906, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.01216509472578764, |
| "rewards/margins": 0.005259203724563122, |
| "rewards/rejected": 0.006905891001224518, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 2.222222222222222e-06, |
| "logits/chosen": -2.731226682662964, |
| "logits/rejected": -2.5713067054748535, |
| "logps/chosen": -256.9880676269531, |
| "logps/rejected": -214.75692749023438, |
| "loss": 0.6895, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.013420146889984608, |
| "rewards/margins": 0.007420173846185207, |
| "rewards/rejected": 0.005999973509460688, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.2875816993464053e-06, |
| "logits/chosen": -2.7683897018432617, |
| "logits/rejected": -2.619065761566162, |
| "logps/chosen": -305.935546875, |
| "logps/rejected": -253.32894897460938, |
| "loss": 0.688, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.018984589725732803, |
| "rewards/margins": 0.010493551380932331, |
| "rewards/rejected": 0.008491038344800472, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.3529411764705885e-06, |
| "logits/chosen": -2.7598016262054443, |
| "logits/rejected": -2.6604552268981934, |
| "logps/chosen": -247.54931640625, |
| "logps/rejected": -245.3474578857422, |
| "loss": 0.6894, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.0182194784283638, |
| "rewards/margins": 0.0077258930541574955, |
| "rewards/rejected": 0.010493585839867592, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.4183006535947716e-06, |
| "logits/chosen": -2.6800496578216553, |
| "logits/rejected": -2.684868812561035, |
| "logps/chosen": -250.73703002929688, |
| "logps/rejected": -229.0603485107422, |
| "loss": 0.6881, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.021059587597846985, |
| "rewards/margins": 0.01036703772842884, |
| "rewards/rejected": 0.010692549869418144, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.4836601307189544e-06, |
| "logits/chosen": -2.7361929416656494, |
| "logits/rejected": -2.643367290496826, |
| "logps/chosen": -272.9651794433594, |
| "logps/rejected": -239.2753143310547, |
| "loss": 0.6851, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.025732994079589844, |
| "rewards/margins": 0.01641467772424221, |
| "rewards/rejected": 0.009318319149315357, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.549019607843137e-06, |
| "logits/chosen": -2.7081408500671387, |
| "logits/rejected": -2.5116286277770996, |
| "logps/chosen": -286.8047790527344, |
| "logps/rejected": -231.5174560546875, |
| "loss": 0.6823, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": 0.03527464345097542, |
| "rewards/margins": 0.02219114825129509, |
| "rewards/rejected": 0.013083499856293201, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.6143790849673208e-06, |
| "logits/chosen": -2.7479846477508545, |
| "logits/rejected": -2.771252155303955, |
| "logps/chosen": -272.4174499511719, |
| "logps/rejected": -283.92108154296875, |
| "loss": 0.6876, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.03528792783617973, |
| "rewards/margins": 0.011549949645996094, |
| "rewards/rejected": 0.02373797819018364, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.05, |
| "eval_logits/chosen": -2.522955894470215, |
| "eval_logits/rejected": -2.4351353645324707, |
| "eval_logps/chosen": -264.435302734375, |
| "eval_logps/rejected": -246.30886840820312, |
| "eval_loss": 0.6845206022262573, |
| "eval_rewards/accuracies": 0.6579999923706055, |
| "eval_rewards/chosen": 0.04054699465632439, |
| "eval_rewards/margins": 0.017824340611696243, |
| "eval_rewards/rejected": 0.022722657769918442, |
| "eval_runtime": 1160.5462, |
| "eval_samples_per_second": 1.723, |
| "eval_steps_per_second": 0.862, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.6797385620915036e-06, |
| "logits/chosen": -2.6627039909362793, |
| "logits/rejected": -2.5389530658721924, |
| "logps/chosen": -244.06640625, |
| "logps/rejected": -203.2464599609375, |
| "loss": 0.6848, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.040567509829998016, |
| "rewards/margins": 0.01748758926987648, |
| "rewards/rejected": 0.023079920560121536, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 2.7450980392156867e-06, |
| "logits/chosen": -2.6784210205078125, |
| "logits/rejected": -2.6046371459960938, |
| "logps/chosen": -261.45135498046875, |
| "logps/rejected": -262.4620666503906, |
| "loss": 0.6811, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.049992240965366364, |
| "rewards/margins": 0.024862922728061676, |
| "rewards/rejected": 0.025129318237304688, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 2.8104575163398695e-06, |
| "logits/chosen": -2.731562852859497, |
| "logits/rejected": -2.607314109802246, |
| "logps/chosen": -264.93365478515625, |
| "logps/rejected": -252.49789428710938, |
| "loss": 0.6815, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.04952297732234001, |
| "rewards/margins": 0.02442570962011814, |
| "rewards/rejected": 0.02509726583957672, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 2.8758169934640523e-06, |
| "logits/chosen": -2.7449309825897217, |
| "logits/rejected": -2.7042744159698486, |
| "logps/chosen": -255.37319946289062, |
| "logps/rejected": -230.885009765625, |
| "loss": 0.6847, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.05002352595329285, |
| "rewards/margins": 0.017672471702098846, |
| "rewards/rejected": 0.032351054251194, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 2.9411764705882355e-06, |
| "logits/chosen": -2.7228033542633057, |
| "logits/rejected": -2.7313358783721924, |
| "logps/chosen": -272.48297119140625, |
| "logps/rejected": -288.769287109375, |
| "loss": 0.684, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.05112846940755844, |
| "rewards/margins": 0.019452670589089394, |
| "rewards/rejected": 0.031675804406404495, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 3.0065359477124182e-06, |
| "logits/chosen": -2.6720528602600098, |
| "logits/rejected": -2.554985761642456, |
| "logps/chosen": -238.60134887695312, |
| "logps/rejected": -231.50741577148438, |
| "loss": 0.6795, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.06515996903181076, |
| "rewards/margins": 0.028682807460427284, |
| "rewards/rejected": 0.036477167159318924, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 3.071895424836602e-06, |
| "logits/chosen": -2.6872806549072266, |
| "logits/rejected": -2.650778293609619, |
| "logps/chosen": -258.9559020996094, |
| "logps/rejected": -241.73696899414062, |
| "loss": 0.6749, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.0744255781173706, |
| "rewards/margins": 0.038194023072719574, |
| "rewards/rejected": 0.03623156249523163, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 3.1372549019607846e-06, |
| "logits/chosen": -2.6812539100646973, |
| "logits/rejected": -2.6287622451782227, |
| "logps/chosen": -267.0392761230469, |
| "logps/rejected": -225.1342010498047, |
| "loss": 0.6688, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": 0.07361556589603424, |
| "rewards/margins": 0.051149915903806686, |
| "rewards/rejected": 0.022465649992227554, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 3.2026143790849674e-06, |
| "logits/chosen": -2.7315773963928223, |
| "logits/rejected": -2.561474561691284, |
| "logps/chosen": -257.4845886230469, |
| "logps/rejected": -213.83639526367188, |
| "loss": 0.6674, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.05385025218129158, |
| "rewards/margins": 0.05505413934588432, |
| "rewards/rejected": -0.0012038892600685358, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.2679738562091506e-06, |
| "logits/chosen": -2.6467950344085693, |
| "logits/rejected": -2.5683364868164062, |
| "logps/chosen": -237.36563110351562, |
| "logps/rejected": -221.3732452392578, |
| "loss": 0.6799, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.02083980292081833, |
| "rewards/margins": 0.029936185106635094, |
| "rewards/rejected": -0.009096382185816765, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.07, |
| "eval_logits/chosen": -2.4660205841064453, |
| "eval_logits/rejected": -2.375483989715576, |
| "eval_logps/chosen": -264.949462890625, |
| "eval_logps/rejected": -249.92762756347656, |
| "eval_loss": 0.6707118153572083, |
| "eval_rewards/accuracies": 0.6815000176429749, |
| "eval_rewards/chosen": 0.035405587404966354, |
| "eval_rewards/margins": 0.04887029901146889, |
| "eval_rewards/rejected": -0.013464723713696003, |
| "eval_runtime": 1161.9283, |
| "eval_samples_per_second": 1.721, |
| "eval_steps_per_second": 0.861, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.3333333333333333e-06, |
| "logits/chosen": -2.557415008544922, |
| "logits/rejected": -2.587461471557617, |
| "logps/chosen": -248.0208740234375, |
| "logps/rejected": -248.16250610351562, |
| "loss": 0.676, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.02980933152139187, |
| "rewards/margins": 0.03878789395093918, |
| "rewards/rejected": -0.008978564292192459, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.398692810457517e-06, |
| "logits/chosen": -2.5861740112304688, |
| "logits/rejected": -2.4508297443389893, |
| "logps/chosen": -242.4407196044922, |
| "logps/rejected": -246.52182006835938, |
| "loss": 0.6693, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 0.01556556485593319, |
| "rewards/margins": 0.05277082324028015, |
| "rewards/rejected": -0.03720525652170181, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.4640522875816997e-06, |
| "logits/chosen": -2.627671957015991, |
| "logits/rejected": -2.5770435333251953, |
| "logps/chosen": -247.16738891601562, |
| "logps/rejected": -231.6279754638672, |
| "loss": 0.6649, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.014550316147506237, |
| "rewards/margins": 0.06174767017364502, |
| "rewards/rejected": -0.047197360545396805, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.529411764705883e-06, |
| "logits/chosen": -2.699380874633789, |
| "logits/rejected": -2.5035440921783447, |
| "logps/chosen": -272.3833923339844, |
| "logps/rejected": -255.84744262695312, |
| "loss": 0.6618, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.0020259805023670197, |
| "rewards/margins": 0.06897115707397461, |
| "rewards/rejected": -0.06694517284631729, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.5947712418300657e-06, |
| "logits/chosen": -2.6517231464385986, |
| "logits/rejected": -2.578498125076294, |
| "logps/chosen": -242.3655548095703, |
| "logps/rejected": -233.637939453125, |
| "loss": 0.6476, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": 0.020458657294511795, |
| "rewards/margins": 0.10107719898223877, |
| "rewards/rejected": -0.08061854541301727, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.6601307189542484e-06, |
| "logits/chosen": -2.6894688606262207, |
| "logits/rejected": -2.506308078765869, |
| "logps/chosen": -277.38946533203125, |
| "logps/rejected": -297.1171569824219, |
| "loss": 0.6589, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.031552791595458984, |
| "rewards/margins": 0.07832719385623932, |
| "rewards/rejected": -0.10987997055053711, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 3.7254901960784316e-06, |
| "logits/chosen": -2.7173991203308105, |
| "logits/rejected": -2.6116385459899902, |
| "logps/chosen": -262.6662902832031, |
| "logps/rejected": -255.9048309326172, |
| "loss": 0.6584, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.06762684136629105, |
| "rewards/margins": 0.077293761074543, |
| "rewards/rejected": -0.14492060244083405, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 3.7908496732026144e-06, |
| "logits/chosen": -2.681260347366333, |
| "logits/rejected": -2.4548838138580322, |
| "logps/chosen": -302.3983154296875, |
| "logps/rejected": -283.51959228515625, |
| "loss": 0.6385, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.025161724537611008, |
| "rewards/margins": 0.125971257686615, |
| "rewards/rejected": -0.1511329710483551, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 3.856209150326798e-06, |
| "logits/chosen": -2.656480312347412, |
| "logits/rejected": -2.4974989891052246, |
| "logps/chosen": -286.38043212890625, |
| "logps/rejected": -257.6932678222656, |
| "loss": 0.6432, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.05199650675058365, |
| "rewards/margins": 0.11849125474691391, |
| "rewards/rejected": -0.17048776149749756, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 3.92156862745098e-06, |
| "logits/chosen": -2.5832934379577637, |
| "logits/rejected": -2.4160356521606445, |
| "logps/chosen": -245.601806640625, |
| "logps/rejected": -255.2142333984375, |
| "loss": 0.6577, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.1271064281463623, |
| "rewards/margins": 0.09216101467609406, |
| "rewards/rejected": -0.21926744282245636, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.08, |
| "eval_logits/chosen": -2.360114097595215, |
| "eval_logits/rejected": -2.2541239261627197, |
| "eval_logps/chosen": -280.78851318359375, |
| "eval_logps/rejected": -272.36041259765625, |
| "eval_loss": 0.6461644172668457, |
| "eval_rewards/accuracies": 0.675000011920929, |
| "eval_rewards/chosen": -0.12298478186130524, |
| "eval_rewards/margins": 0.11480776965618134, |
| "eval_rewards/rejected": -0.23779255151748657, |
| "eval_runtime": 1158.9282, |
| "eval_samples_per_second": 1.726, |
| "eval_steps_per_second": 0.863, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 3.986928104575164e-06, |
| "logits/chosen": -2.62365460395813, |
| "logits/rejected": -2.392054557800293, |
| "logps/chosen": -231.8965606689453, |
| "logps/rejected": -213.38131713867188, |
| "loss": 0.6325, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.05727694183588028, |
| "rewards/margins": 0.13814638555049896, |
| "rewards/rejected": -0.19542333483695984, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 4.052287581699347e-06, |
| "logits/chosen": -2.64388108253479, |
| "logits/rejected": -2.4387760162353516, |
| "logps/chosen": -285.2777404785156, |
| "logps/rejected": -278.3298645019531, |
| "loss": 0.6246, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.07192676514387131, |
| "rewards/margins": 0.16072218120098114, |
| "rewards/rejected": -0.23264892399311066, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 4.11764705882353e-06, |
| "logits/chosen": -2.4769465923309326, |
| "logits/rejected": -2.419826030731201, |
| "logps/chosen": -278.97564697265625, |
| "logps/rejected": -272.57269287109375, |
| "loss": 0.6501, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.16124705970287323, |
| "rewards/margins": 0.11454138904809952, |
| "rewards/rejected": -0.27578845620155334, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 4.183006535947713e-06, |
| "logits/chosen": -2.6546480655670166, |
| "logits/rejected": -2.4742162227630615, |
| "logps/chosen": -272.10870361328125, |
| "logps/rejected": -273.37518310546875, |
| "loss": 0.6526, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.15491154789924622, |
| "rewards/margins": 0.1191616877913475, |
| "rewards/rejected": -0.27407321333885193, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.2483660130718954e-06, |
| "logits/chosen": -2.5434234142303467, |
| "logits/rejected": -2.544323682785034, |
| "logps/chosen": -271.45587158203125, |
| "logps/rejected": -271.0767517089844, |
| "loss": 0.6503, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.1807040274143219, |
| "rewards/margins": 0.1250898838043213, |
| "rewards/rejected": -0.3057938814163208, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.313725490196079e-06, |
| "logits/chosen": -2.596226215362549, |
| "logits/rejected": -2.439690113067627, |
| "logps/chosen": -318.5782775878906, |
| "logps/rejected": -278.7407531738281, |
| "loss": 0.6247, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.22762902081012726, |
| "rewards/margins": 0.16828063130378723, |
| "rewards/rejected": -0.3959096074104309, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.379084967320262e-06, |
| "logits/chosen": -2.6381888389587402, |
| "logits/rejected": -2.522258758544922, |
| "logps/chosen": -300.07244873046875, |
| "logps/rejected": -334.3465270996094, |
| "loss": 0.6205, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.18682761490345, |
| "rewards/margins": 0.18621641397476196, |
| "rewards/rejected": -0.3730439841747284, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.444444444444444e-06, |
| "logits/chosen": -2.663062572479248, |
| "logits/rejected": -2.5052947998046875, |
| "logps/chosen": -281.8200988769531, |
| "logps/rejected": -277.8845520019531, |
| "loss": 0.6352, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.13653087615966797, |
| "rewards/margins": 0.1511278599500656, |
| "rewards/rejected": -0.2876587510108948, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.509803921568628e-06, |
| "logits/chosen": -2.664280652999878, |
| "logits/rejected": -2.524028778076172, |
| "logps/chosen": -309.06658935546875, |
| "logps/rejected": -282.0691223144531, |
| "loss": 0.6352, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.15293274819850922, |
| "rewards/margins": 0.15575169026851654, |
| "rewards/rejected": -0.30868446826934814, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.5751633986928105e-06, |
| "logits/chosen": -2.6197104454040527, |
| "logits/rejected": -2.5259227752685547, |
| "logps/chosen": -279.31195068359375, |
| "logps/rejected": -282.20111083984375, |
| "loss": 0.6365, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.11969868838787079, |
| "rewards/margins": 0.15021947026252747, |
| "rewards/rejected": -0.26991817355155945, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.09, |
| "eval_logits/chosen": -2.31355619430542, |
| "eval_logits/rejected": -2.2012522220611572, |
| "eval_logps/chosen": -277.0453186035156, |
| "eval_logps/rejected": -272.20367431640625, |
| "eval_loss": 0.6344681978225708, |
| "eval_rewards/accuracies": 0.6859999895095825, |
| "eval_rewards/chosen": -0.08555291593074799, |
| "eval_rewards/margins": 0.15067268908023834, |
| "eval_rewards/rejected": -0.23622561991214752, |
| "eval_runtime": 1159.7861, |
| "eval_samples_per_second": 1.724, |
| "eval_steps_per_second": 0.862, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.640522875816994e-06, |
| "logits/chosen": -2.5975005626678467, |
| "logits/rejected": -2.508746862411499, |
| "logps/chosen": -281.31512451171875, |
| "logps/rejected": -281.1041259765625, |
| "loss": 0.6322, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.062341589480638504, |
| "rewards/margins": 0.1610972285270691, |
| "rewards/rejected": -0.2234388291835785, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 4.705882352941177e-06, |
| "logits/chosen": -2.6123766899108887, |
| "logits/rejected": -2.5293030738830566, |
| "logps/chosen": -323.938720703125, |
| "logps/rejected": -315.38092041015625, |
| "loss": 0.624, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.09480178356170654, |
| "rewards/margins": 0.18506909906864166, |
| "rewards/rejected": -0.2798708975315094, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.77124183006536e-06, |
| "logits/chosen": -2.605785369873047, |
| "logits/rejected": -2.518705129623413, |
| "logps/chosen": -289.82965087890625, |
| "logps/rejected": -295.834228515625, |
| "loss": 0.6195, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.12233264744281769, |
| "rewards/margins": 0.18506425619125366, |
| "rewards/rejected": -0.30739688873291016, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.836601307189543e-06, |
| "logits/chosen": -2.632423162460327, |
| "logits/rejected": -2.4382903575897217, |
| "logps/chosen": -304.190185546875, |
| "logps/rejected": -279.628662109375, |
| "loss": 0.6155, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.10397078841924667, |
| "rewards/margins": 0.20101885497570038, |
| "rewards/rejected": -0.30498963594436646, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.901960784313726e-06, |
| "logits/chosen": -2.450904369354248, |
| "logits/rejected": -2.3369948863983154, |
| "logps/chosen": -257.42974853515625, |
| "logps/rejected": -265.8070373535156, |
| "loss": 0.6048, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.1934954673051834, |
| "rewards/margins": 0.22951745986938477, |
| "rewards/rejected": -0.423012912273407, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.967320261437909e-06, |
| "logits/chosen": -2.5839266777038574, |
| "logits/rejected": -2.357056140899658, |
| "logps/chosen": -303.64654541015625, |
| "logps/rejected": -279.87774658203125, |
| "loss": 0.6236, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.32853788137435913, |
| "rewards/margins": 0.19128181040287018, |
| "rewards/rejected": -0.5198196172714233, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.999993476542427e-06, |
| "logits/chosen": -2.515817165374756, |
| "logits/rejected": -2.486161708831787, |
| "logps/chosen": -327.0534973144531, |
| "logps/rejected": -329.8384704589844, |
| "loss": 0.6198, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.4592045247554779, |
| "rewards/margins": 0.23688983917236328, |
| "rewards/rejected": -0.6960943937301636, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.999941289086112e-06, |
| "logits/chosen": -2.5871434211730957, |
| "logits/rejected": -2.2986748218536377, |
| "logps/chosen": -352.54925537109375, |
| "logps/rejected": -345.39727783203125, |
| "loss": 0.602, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.615364134311676, |
| "rewards/margins": 0.27003955841064453, |
| "rewards/rejected": -0.8854037523269653, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.999836915262896e-06, |
| "logits/chosen": -2.4022507667541504, |
| "logits/rejected": -2.423859119415283, |
| "logps/chosen": -341.8403625488281, |
| "logps/rejected": -363.97869873046875, |
| "loss": 0.6102, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.6583345532417297, |
| "rewards/margins": 0.2388831377029419, |
| "rewards/rejected": -0.8972176313400269, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 4.999680357251587e-06, |
| "logits/chosen": -2.2201342582702637, |
| "logits/rejected": -2.259474277496338, |
| "logps/chosen": -311.77374267578125, |
| "logps/rejected": -336.84295654296875, |
| "loss": 0.6519, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.613008975982666, |
| "rewards/margins": 0.15706071257591248, |
| "rewards/rejected": -0.7700697183609009, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.1, |
| "eval_logits/chosen": -2.183530330657959, |
| "eval_logits/rejected": -2.04819917678833, |
| "eval_logps/chosen": -317.9223327636719, |
| "eval_logps/rejected": -320.8871765136719, |
| "eval_loss": 0.6239581108093262, |
| "eval_rewards/accuracies": 0.6629999876022339, |
| "eval_rewards/chosen": -0.49432334303855896, |
| "eval_rewards/margins": 0.22873705625534058, |
| "eval_rewards/rejected": -0.7230603098869324, |
| "eval_runtime": 1149.5976, |
| "eval_samples_per_second": 1.74, |
| "eval_steps_per_second": 0.87, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.999471618320339e-06, |
| "logits/chosen": -2.5523288249969482, |
| "logits/rejected": -2.320265054702759, |
| "logps/chosen": -328.2205810546875, |
| "logps/rejected": -322.53076171875, |
| "loss": 0.6069, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.4844594895839691, |
| "rewards/margins": 0.2521725296974182, |
| "rewards/rejected": -0.7366319894790649, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.999210702826586e-06, |
| "logits/chosen": -2.6358237266540527, |
| "logits/rejected": -2.4474587440490723, |
| "logps/chosen": -346.6995849609375, |
| "logps/rejected": -324.92242431640625, |
| "loss": 0.6221, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.4281977117061615, |
| "rewards/margins": 0.23045018315315247, |
| "rewards/rejected": -0.658647894859314, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.998897616216947e-06, |
| "logits/chosen": -2.4511475563049316, |
| "logits/rejected": -2.5670619010925293, |
| "logps/chosen": -258.81573486328125, |
| "logps/rejected": -311.01043701171875, |
| "loss": 0.6131, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.3054274618625641, |
| "rewards/margins": 0.24654905498027802, |
| "rewards/rejected": -0.5519765019416809, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.998532365027117e-06, |
| "logits/chosen": -2.4918437004089355, |
| "logits/rejected": -2.2926580905914307, |
| "logps/chosen": -311.8231201171875, |
| "logps/rejected": -280.79620361328125, |
| "loss": 0.5902, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.2812344431877136, |
| "rewards/margins": 0.2974868416786194, |
| "rewards/rejected": -0.5787213444709778, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.9981149568817275e-06, |
| "logits/chosen": -2.498668909072876, |
| "logits/rejected": -2.4396538734436035, |
| "logps/chosen": -313.8962707519531, |
| "logps/rejected": -356.46392822265625, |
| "loss": 0.6126, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.4118213653564453, |
| "rewards/margins": 0.26523926854133606, |
| "rewards/rejected": -0.677060604095459, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.997645400494192e-06, |
| "logits/chosen": -2.544689178466797, |
| "logits/rejected": -2.427544116973877, |
| "logps/chosen": -297.4951477050781, |
| "logps/rejected": -325.6196594238281, |
| "loss": 0.6211, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.5930116176605225, |
| "rewards/margins": 0.29512810707092285, |
| "rewards/rejected": -0.8881398439407349, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 4.997123705666514e-06, |
| "logits/chosen": -2.5834832191467285, |
| "logits/rejected": -2.3967490196228027, |
| "logps/chosen": -340.9512939453125, |
| "logps/rejected": -351.071044921875, |
| "loss": 0.6576, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.6529733538627625, |
| "rewards/margins": 0.1915055215358734, |
| "rewards/rejected": -0.8444789052009583, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.996549883289093e-06, |
| "logits/chosen": -2.5025086402893066, |
| "logits/rejected": -2.4052650928497314, |
| "logps/chosen": -308.8294677734375, |
| "logps/rejected": -350.6068115234375, |
| "loss": 0.6265, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.593645453453064, |
| "rewards/margins": 0.27240580320358276, |
| "rewards/rejected": -0.866051197052002, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.995923945340495e-06, |
| "logits/chosen": -2.5428309440612793, |
| "logits/rejected": -2.4630656242370605, |
| "logps/chosen": -305.7501525878906, |
| "logps/rejected": -334.24639892578125, |
| "loss": 0.6473, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.537492036819458, |
| "rewards/margins": 0.21050593256950378, |
| "rewards/rejected": -0.7479979395866394, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.995245904887195e-06, |
| "logits/chosen": -2.57853364944458, |
| "logits/rejected": -2.3580322265625, |
| "logps/chosen": -310.814208984375, |
| "logps/rejected": -298.79962158203125, |
| "loss": 0.6547, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.7046155333518982, |
| "rewards/margins": 0.197780042886734, |
| "rewards/rejected": -0.9023955464363098, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.12, |
| "eval_logits/chosen": -2.2183713912963867, |
| "eval_logits/rejected": -2.0783114433288574, |
| "eval_logps/chosen": -325.8177490234375, |
| "eval_logps/rejected": -331.4542236328125, |
| "eval_loss": 0.6203334927558899, |
| "eval_rewards/accuracies": 0.6694999933242798, |
| "eval_rewards/chosen": -0.5732770562171936, |
| "eval_rewards/margins": 0.25545385479927063, |
| "eval_rewards/rejected": -0.8287308216094971, |
| "eval_runtime": 1154.0508, |
| "eval_samples_per_second": 1.733, |
| "eval_steps_per_second": 0.867, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.994515776083313e-06, |
| "logits/chosen": -2.4050710201263428, |
| "logits/rejected": -2.4928653240203857, |
| "logps/chosen": -315.0536193847656, |
| "logps/rejected": -373.6515197753906, |
| "loss": 0.5972, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.47445574402809143, |
| "rewards/margins": 0.32963109016418457, |
| "rewards/rejected": -0.8040868639945984, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.993733574170316e-06, |
| "logits/chosen": -2.5724964141845703, |
| "logits/rejected": -2.402193546295166, |
| "logps/chosen": -261.9773864746094, |
| "logps/rejected": -292.7881774902344, |
| "loss": 0.5911, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3413178622722626, |
| "rewards/margins": 0.32790833711624146, |
| "rewards/rejected": -0.6692262291908264, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.992899315476696e-06, |
| "logits/chosen": -2.6308352947235107, |
| "logits/rejected": -2.504096508026123, |
| "logps/chosen": -331.2294616699219, |
| "logps/rejected": -336.93280029296875, |
| "loss": 0.5964, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.303416907787323, |
| "rewards/margins": 0.3124091327190399, |
| "rewards/rejected": -0.6158260703086853, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.9920130174176354e-06, |
| "logits/chosen": -2.5981762409210205, |
| "logits/rejected": -2.4487905502319336, |
| "logps/chosen": -319.9151916503906, |
| "logps/rejected": -333.0742492675781, |
| "loss": 0.5814, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.42610105872154236, |
| "rewards/margins": 0.30205437541007996, |
| "rewards/rejected": -0.7281554341316223, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 4.991074698494638e-06, |
| "logits/chosen": -2.6599678993225098, |
| "logits/rejected": -2.3881497383117676, |
| "logps/chosen": -321.3490905761719, |
| "logps/rejected": -312.3178405761719, |
| "loss": 0.6153, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.4955690801143646, |
| "rewards/margins": 0.26488804817199707, |
| "rewards/rejected": -0.7604571580886841, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.990084378295148e-06, |
| "logits/chosen": -2.6287171840667725, |
| "logits/rejected": -2.499150514602661, |
| "logps/chosen": -286.4674987792969, |
| "logps/rejected": -286.6602478027344, |
| "loss": 0.6153, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.46585598587989807, |
| "rewards/margins": 0.2714241147041321, |
| "rewards/rejected": -0.737280011177063, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.989042077492135e-06, |
| "logits/chosen": -2.6049046516418457, |
| "logits/rejected": -2.512279987335205, |
| "logps/chosen": -312.2615051269531, |
| "logps/rejected": -334.3189392089844, |
| "loss": 0.548, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.38576096296310425, |
| "rewards/margins": 0.3811526596546173, |
| "rewards/rejected": -0.7669135928153992, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.987947817843665e-06, |
| "logits/chosen": -2.4653379917144775, |
| "logits/rejected": -2.4543960094451904, |
| "logps/chosen": -293.2818908691406, |
| "logps/rejected": -294.23089599609375, |
| "loss": 0.6096, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.46745139360427856, |
| "rewards/margins": 0.312784880399704, |
| "rewards/rejected": -0.7802362442016602, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.986801622192453e-06, |
| "logits/chosen": -2.585569381713867, |
| "logits/rejected": -2.398247241973877, |
| "logps/chosen": -264.38580322265625, |
| "logps/rejected": -286.49078369140625, |
| "loss": 0.5645, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.41495591402053833, |
| "rewards/margins": 0.42753225564956665, |
| "rewards/rejected": -0.842488169670105, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.985603514465372e-06, |
| "logits/chosen": -2.496156692504883, |
| "logits/rejected": -2.562666416168213, |
| "logps/chosen": -313.02593994140625, |
| "logps/rejected": -339.96746826171875, |
| "loss": 0.5841, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.47666803002357483, |
| "rewards/margins": 0.3657141327857971, |
| "rewards/rejected": -0.8423821330070496, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.13, |
| "eval_logits/chosen": -2.2086312770843506, |
| "eval_logits/rejected": -2.068852186203003, |
| "eval_logps/chosen": -322.0998229980469, |
| "eval_logps/rejected": -334.5815734863281, |
| "eval_loss": 0.6070671677589417, |
| "eval_rewards/accuracies": 0.6819999814033508, |
| "eval_rewards/chosen": -0.5360978245735168, |
| "eval_rewards/margins": 0.32390668988227844, |
| "eval_rewards/rejected": -0.8600045442581177, |
| "eval_runtime": 1162.8161, |
| "eval_samples_per_second": 1.72, |
| "eval_steps_per_second": 0.86, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.984353519672966e-06, |
| "logits/chosen": -2.5102970600128174, |
| "logits/rejected": -2.296638011932373, |
| "logps/chosen": -322.3448486328125, |
| "logps/rejected": -314.18927001953125, |
| "loss": 0.652, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.6231977343559265, |
| "rewards/margins": 0.2020292580127716, |
| "rewards/rejected": -0.8252270817756653, |
| "step": 1010 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.9830516639089226e-06, |
| "logits/chosen": -2.5087199211120605, |
| "logits/rejected": -2.2799477577209473, |
| "logps/chosen": -359.6461486816406, |
| "logps/rejected": -333.55712890625, |
| "loss": 0.5308, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.509143590927124, |
| "rewards/margins": 0.46995463967323303, |
| "rewards/rejected": -0.9790982007980347, |
| "step": 1020 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.9816979743495296e-06, |
| "logits/chosen": -2.437312602996826, |
| "logits/rejected": -2.312748670578003, |
| "logps/chosen": -381.3270263671875, |
| "logps/rejected": -398.35748291015625, |
| "loss": 0.5985, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.8338835835456848, |
| "rewards/margins": 0.41778063774108887, |
| "rewards/rejected": -1.251664400100708, |
| "step": 1030 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.980292479253105e-06, |
| "logits/chosen": -2.523524761199951, |
| "logits/rejected": -2.1683216094970703, |
| "logps/chosen": -413.48956298828125, |
| "logps/rejected": -424.48797607421875, |
| "loss": 0.5374, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.087397575378418, |
| "rewards/margins": 0.5586522817611694, |
| "rewards/rejected": -1.6460498571395874, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.978835207959414e-06, |
| "logits/chosen": -2.291563034057617, |
| "logits/rejected": -2.2155869007110596, |
| "logps/chosen": -365.6241760253906, |
| "logps/rejected": -386.0607604980469, |
| "loss": 0.5994, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.0963432788848877, |
| "rewards/margins": 0.34998494386672974, |
| "rewards/rejected": -1.4463282823562622, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.977326190889046e-06, |
| "logits/chosen": -2.4078869819641113, |
| "logits/rejected": -1.7181726694107056, |
| "logps/chosen": -359.93756103515625, |
| "logps/rejected": -351.11859130859375, |
| "loss": 0.5669, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.9058128595352173, |
| "rewards/margins": 0.4577213227748871, |
| "rewards/rejected": -1.3635342121124268, |
| "step": 1060 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.975765459542788e-06, |
| "logits/chosen": -2.2240052223205566, |
| "logits/rejected": -2.059451103210449, |
| "logps/chosen": -315.9155578613281, |
| "logps/rejected": -355.5037841796875, |
| "loss": 0.528, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.5928197503089905, |
| "rewards/margins": 0.5360963344573975, |
| "rewards/rejected": -1.1289160251617432, |
| "step": 1070 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.9741530465009665e-06, |
| "logits/chosen": -2.186079502105713, |
| "logits/rejected": -1.9260094165802002, |
| "logps/chosen": -312.0130920410156, |
| "logps/rejected": -334.43389892578125, |
| "loss": 0.5806, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.6744036078453064, |
| "rewards/margins": 0.40451207756996155, |
| "rewards/rejected": -1.0789155960083008, |
| "step": 1080 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.972488985422763e-06, |
| "logits/chosen": -2.0618720054626465, |
| "logits/rejected": -1.9399276971817017, |
| "logps/chosen": -329.8536376953125, |
| "logps/rejected": -354.6712341308594, |
| "loss": 0.5418, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.8148940205574036, |
| "rewards/margins": 0.6003459692001343, |
| "rewards/rejected": -1.4152400493621826, |
| "step": 1090 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 4.970773311045514e-06, |
| "logits/chosen": -2.120664358139038, |
| "logits/rejected": -1.6714054346084595, |
| "logps/chosen": -371.4166564941406, |
| "logps/rejected": -393.62786865234375, |
| "loss": 0.5877, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.0566858053207397, |
| "rewards/margins": 0.43407002091407776, |
| "rewards/rejected": -1.4907559156417847, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.14, |
| "eval_logits/chosen": -1.3836040496826172, |
| "eval_logits/rejected": -1.1052757501602173, |
| "eval_logps/chosen": -383.43798828125, |
| "eval_logps/rejected": -410.8677673339844, |
| "eval_loss": 0.5946979522705078, |
| "eval_rewards/accuracies": 0.6855000257492065, |
| "eval_rewards/chosen": -1.149479866027832, |
| "eval_rewards/margins": 0.4733865261077881, |
| "eval_rewards/rejected": -1.6228665113449097, |
| "eval_runtime": 1178.8626, |
| "eval_samples_per_second": 1.697, |
| "eval_steps_per_second": 0.848, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.969006059183984e-06, |
| "logits/chosen": -2.1172611713409424, |
| "logits/rejected": -1.6654752492904663, |
| "logps/chosen": -387.62664794921875, |
| "logps/rejected": -408.0655212402344, |
| "loss": 0.6262, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.2082122564315796, |
| "rewards/margins": 0.43365636467933655, |
| "rewards/rejected": -1.6418688297271729, |
| "step": 1110 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.967187266729623e-06, |
| "logits/chosen": -2.193946123123169, |
| "logits/rejected": -1.850895643234253, |
| "logps/chosen": -398.3614501953125, |
| "logps/rejected": -430.51373291015625, |
| "loss": 0.5971, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.2287832498550415, |
| "rewards/margins": 0.43036943674087524, |
| "rewards/rejected": -1.6591527462005615, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.965316971649791e-06, |
| "logits/chosen": -2.100188732147217, |
| "logits/rejected": -1.686700463294983, |
| "logps/chosen": -396.20623779296875, |
| "logps/rejected": -422.5747985839844, |
| "loss": 0.4954, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.049180269241333, |
| "rewards/margins": 0.7083006501197815, |
| "rewards/rejected": -1.7574809789657593, |
| "step": 1130 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.963395212986964e-06, |
| "logits/chosen": -2.071812152862549, |
| "logits/rejected": -1.535829782485962, |
| "logps/chosen": -342.7199401855469, |
| "logps/rejected": -373.0933837890625, |
| "loss": 0.526, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.0833193063735962, |
| "rewards/margins": 0.5862609148025513, |
| "rewards/rejected": -1.6695802211761475, |
| "step": 1140 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.9614220308579285e-06, |
| "logits/chosen": -1.8345826864242554, |
| "logits/rejected": -2.1796462535858154, |
| "logps/chosen": -351.6676330566406, |
| "logps/rejected": -381.9559326171875, |
| "loss": 0.5963, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.7989309430122375, |
| "rewards/margins": 0.41886311769485474, |
| "rewards/rejected": -1.2177939414978027, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.9593974664529325e-06, |
| "logits/chosen": -2.1840920448303223, |
| "logits/rejected": -1.7762985229492188, |
| "logps/chosen": -331.3350830078125, |
| "logps/rejected": -385.296630859375, |
| "loss": 0.5442, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.6507974863052368, |
| "rewards/margins": 0.5434342622756958, |
| "rewards/rejected": -1.1942317485809326, |
| "step": 1160 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.957321562034833e-06, |
| "logits/chosen": -2.0527548789978027, |
| "logits/rejected": -1.8421388864517212, |
| "logps/chosen": -366.1250305175781, |
| "logps/rejected": -394.0170593261719, |
| "loss": 0.559, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.8194394111633301, |
| "rewards/margins": 0.5880136489868164, |
| "rewards/rejected": -1.4074528217315674, |
| "step": 1170 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 4.955194360938214e-06, |
| "logits/chosen": -1.9533300399780273, |
| "logits/rejected": -1.7631326913833618, |
| "logps/chosen": -371.4923400878906, |
| "logps/rejected": -387.41253662109375, |
| "loss": 0.5852, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.1551687717437744, |
| "rewards/margins": 0.4826745092868805, |
| "rewards/rejected": -1.637843370437622, |
| "step": 1180 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.9530159075684735e-06, |
| "logits/chosen": -1.4516584873199463, |
| "logits/rejected": -1.5901073217391968, |
| "logps/chosen": -388.7632141113281, |
| "logps/rejected": -484.33258056640625, |
| "loss": 0.6893, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -1.5982259511947632, |
| "rewards/margins": 0.268177330493927, |
| "rewards/rejected": -1.8664032220840454, |
| "step": 1190 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.950786247400908e-06, |
| "logits/chosen": -1.6140592098236084, |
| "logits/rejected": -1.6555970907211304, |
| "logps/chosen": -390.38330078125, |
| "logps/rejected": -435.25604248046875, |
| "loss": 0.5552, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.4886068105697632, |
| "rewards/margins": 0.4878392219543457, |
| "rewards/rejected": -1.9764461517333984, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.16, |
| "eval_logits/chosen": -0.7372294068336487, |
| "eval_logits/rejected": -0.36139780282974243, |
| "eval_logps/chosen": -411.0458679199219, |
| "eval_logps/rejected": -437.91998291015625, |
| "eval_loss": 0.5908603072166443, |
| "eval_rewards/accuracies": 0.6880000233650208, |
| "eval_rewards/chosen": -1.4255588054656982, |
| "eval_rewards/margins": 0.4678295850753784, |
| "eval_rewards/rejected": -1.8933883905410767, |
| "eval_runtime": 1176.6814, |
| "eval_samples_per_second": 1.7, |
| "eval_steps_per_second": 0.85, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.948505426979756e-06, |
| "logits/chosen": -1.8322169780731201, |
| "logits/rejected": -1.5653588771820068, |
| "logps/chosen": -406.3680114746094, |
| "logps/rejected": -444.8448791503906, |
| "loss": 0.5533, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.4452760219573975, |
| "rewards/margins": 0.5475454926490784, |
| "rewards/rejected": -1.992821455001831, |
| "step": 1210 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.946173493917228e-06, |
| "logits/chosen": -1.7810437679290771, |
| "logits/rejected": -1.2556498050689697, |
| "logps/chosen": -417.5589294433594, |
| "logps/rejected": -397.54437255859375, |
| "loss": 0.733, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -1.5235958099365234, |
| "rewards/margins": 0.1696973741054535, |
| "rewards/rejected": -1.6932932138442993, |
| "step": 1220 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.943790496892513e-06, |
| "logits/chosen": -1.9522157907485962, |
| "logits/rejected": -1.3978416919708252, |
| "logps/chosen": -345.17913818359375, |
| "logps/rejected": -371.7323303222656, |
| "loss": 0.5339, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.8955880999565125, |
| "rewards/margins": 0.6181024312973022, |
| "rewards/rejected": -1.5136905908584595, |
| "step": 1230 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.941356485650762e-06, |
| "logits/chosen": -2.062680721282959, |
| "logits/rejected": -1.6688048839569092, |
| "logps/chosen": -390.939208984375, |
| "logps/rejected": -419.36334228515625, |
| "loss": 0.5703, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.8187043070793152, |
| "rewards/margins": 0.4967420995235443, |
| "rewards/rejected": -1.3154462575912476, |
| "step": 1240 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.93887151100205e-06, |
| "logits/chosen": -2.130277156829834, |
| "logits/rejected": -1.685664176940918, |
| "logps/chosen": -385.4061279296875, |
| "logps/rejected": -389.6874084472656, |
| "loss": 0.6048, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.6924458146095276, |
| "rewards/margins": 0.38014930486679077, |
| "rewards/rejected": -1.0725951194763184, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 4.936335624820313e-06, |
| "logits/chosen": -2.035191774368286, |
| "logits/rejected": -1.712632417678833, |
| "logps/chosen": -348.1141052246094, |
| "logps/rejected": -357.98541259765625, |
| "loss": 0.5586, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.8647063970565796, |
| "rewards/margins": 0.4742640554904938, |
| "rewards/rejected": -1.3389705419540405, |
| "step": 1260 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.933748880042271e-06, |
| "logits/chosen": -2.0088629722595215, |
| "logits/rejected": -1.5719774961471558, |
| "logps/chosen": -360.0817565917969, |
| "logps/rejected": -390.73211669921875, |
| "loss": 0.5454, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.9902151226997375, |
| "rewards/margins": 0.534164309501648, |
| "rewards/rejected": -1.5243794918060303, |
| "step": 1270 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.931111330666317e-06, |
| "logits/chosen": -2.0022754669189453, |
| "logits/rejected": -1.2308666706085205, |
| "logps/chosen": -358.9193420410156, |
| "logps/rejected": -354.67352294921875, |
| "loss": 0.6109, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.0700308084487915, |
| "rewards/margins": 0.3465597927570343, |
| "rewards/rejected": -1.4165904521942139, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.9284230317513906e-06, |
| "logits/chosen": -1.9100980758666992, |
| "logits/rejected": -1.4506380558013916, |
| "logps/chosen": -428.6927795410156, |
| "logps/rejected": -433.57489013671875, |
| "loss": 0.5906, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.2611325979232788, |
| "rewards/margins": 0.5384291410446167, |
| "rewards/rejected": -1.7995617389678955, |
| "step": 1290 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.9256840394158325e-06, |
| "logits/chosen": -1.5153193473815918, |
| "logits/rejected": -1.5525444746017456, |
| "logps/chosen": -408.9667053222656, |
| "logps/rejected": -496.4200134277344, |
| "loss": 0.5492, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.3559167385101318, |
| "rewards/margins": 0.5765284895896912, |
| "rewards/rejected": -1.9324451684951782, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.17, |
| "eval_logits/chosen": -0.5949398279190063, |
| "eval_logits/rejected": -0.19331219792366028, |
| "eval_logps/chosen": -414.6322937011719, |
| "eval_logps/rejected": -446.29095458984375, |
| "eval_loss": 0.5791042447090149, |
| "eval_rewards/accuracies": 0.6934999823570251, |
| "eval_rewards/chosen": -1.461422085762024, |
| "eval_rewards/margins": 0.5156759023666382, |
| "eval_rewards/rejected": -1.977097988128662, |
| "eval_runtime": 1198.5552, |
| "eval_samples_per_second": 1.669, |
| "eval_steps_per_second": 0.834, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.922894410836207e-06, |
| "logits/chosen": -1.9784332513809204, |
| "logits/rejected": -1.0070569515228271, |
| "logps/chosen": -438.13262939453125, |
| "logps/rejected": -431.6548767089844, |
| "loss": 0.5808, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.5133016109466553, |
| "rewards/margins": 0.5359851121902466, |
| "rewards/rejected": -2.0492866039276123, |
| "step": 1310 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.920054204246116e-06, |
| "logits/chosen": -1.8840601444244385, |
| "logits/rejected": -1.0693460702896118, |
| "logps/chosen": -434.94189453125, |
| "logps/rejected": -427.7396545410156, |
| "loss": 0.6386, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -1.5475298166275024, |
| "rewards/margins": 0.38071927428245544, |
| "rewards/rejected": -1.9282491207122803, |
| "step": 1320 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 4.9171634789349744e-06, |
| "logits/chosen": -1.7856547832489014, |
| "logits/rejected": -1.456695795059204, |
| "logps/chosen": -393.8877868652344, |
| "logps/rejected": -460.5874938964844, |
| "loss": 0.5115, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.295278787612915, |
| "rewards/margins": 0.6688205003738403, |
| "rewards/rejected": -1.9640991687774658, |
| "step": 1330 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.914222295246782e-06, |
| "logits/chosen": -1.7937772274017334, |
| "logits/rejected": -1.691471815109253, |
| "logps/chosen": -362.4700927734375, |
| "logps/rejected": -393.37750244140625, |
| "loss": 0.6389, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.0026386976242065, |
| "rewards/margins": 0.29950928688049316, |
| "rewards/rejected": -1.3021478652954102, |
| "step": 1340 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.911230714578858e-06, |
| "logits/chosen": -1.6912847757339478, |
| "logits/rejected": -1.8542919158935547, |
| "logps/chosen": -274.45208740234375, |
| "logps/rejected": -340.21807861328125, |
| "loss": 0.5671, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.5728007555007935, |
| "rewards/margins": 0.4466250538825989, |
| "rewards/rejected": -1.0194257497787476, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.908188799380558e-06, |
| "logits/chosen": -2.217525005340576, |
| "logits/rejected": -1.9035638570785522, |
| "logps/chosen": -292.1073303222656, |
| "logps/rejected": -301.85491943359375, |
| "loss": 0.5692, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.46799778938293457, |
| "rewards/margins": 0.4237847924232483, |
| "rewards/rejected": -0.8917825818061829, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.905096613151975e-06, |
| "logits/chosen": -2.0345702171325684, |
| "logits/rejected": -1.6463664770126343, |
| "logps/chosen": -375.4377746582031, |
| "logps/rejected": -376.78997802734375, |
| "loss": 0.629, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.7877501845359802, |
| "rewards/margins": 0.28059062361717224, |
| "rewards/rejected": -1.06834077835083, |
| "step": 1370 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.90195422044261e-06, |
| "logits/chosen": -1.9617021083831787, |
| "logits/rejected": -1.6719785928726196, |
| "logps/chosen": -360.3094787597656, |
| "logps/rejected": -401.42926025390625, |
| "loss": 0.4957, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.7087548971176147, |
| "rewards/margins": 0.6915256381034851, |
| "rewards/rejected": -1.4002805948257446, |
| "step": 1380 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.898761686850028e-06, |
| "logits/chosen": -1.6123435497283936, |
| "logits/rejected": -1.2256313562393188, |
| "logps/chosen": -361.95721435546875, |
| "logps/rejected": -406.7974548339844, |
| "loss": 0.5884, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.009804129600525, |
| "rewards/margins": 0.534344494342804, |
| "rewards/rejected": -1.544148564338684, |
| "step": 1390 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.895519079018485e-06, |
| "logits/chosen": -1.6615076065063477, |
| "logits/rejected": -0.7398694753646851, |
| "logps/chosen": -338.4859619140625, |
| "logps/rejected": -375.15704345703125, |
| "loss": 0.5789, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.8177839517593384, |
| "rewards/margins": 0.6118384599685669, |
| "rewards/rejected": -1.4296224117279053, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.18, |
| "eval_logits/chosen": -0.5845944285392761, |
| "eval_logits/rejected": -0.1907668113708496, |
| "eval_logps/chosen": -356.483154296875, |
| "eval_logps/rejected": -384.9108581542969, |
| "eval_loss": 0.5771133303642273, |
| "eval_rewards/accuracies": 0.703499972820282, |
| "eval_rewards/chosen": -0.8799312114715576, |
| "eval_rewards/margins": 0.4833654463291168, |
| "eval_rewards/rejected": -1.3632965087890625, |
| "eval_runtime": 1177.7993, |
| "eval_samples_per_second": 1.698, |
| "eval_steps_per_second": 0.849, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 4.89222646463754e-06, |
| "logits/chosen": -1.5245163440704346, |
| "logits/rejected": -1.3327850103378296, |
| "logps/chosen": -351.33154296875, |
| "logps/rejected": -393.2879333496094, |
| "loss": 0.5975, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.9374910593032837, |
| "rewards/margins": 0.4914394021034241, |
| "rewards/rejected": -1.4289302825927734, |
| "step": 1410 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.888883912440642e-06, |
| "logits/chosen": -1.7364609241485596, |
| "logits/rejected": -1.2762303352355957, |
| "logps/chosen": -401.25, |
| "logps/rejected": -442.7745666503906, |
| "loss": 0.5555, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.9093993306159973, |
| "rewards/margins": 0.5804132223129272, |
| "rewards/rejected": -1.4898124933242798, |
| "step": 1420 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.885491492203688e-06, |
| "logits/chosen": -1.50997793674469, |
| "logits/rejected": -1.1342096328735352, |
| "logps/chosen": -360.4520568847656, |
| "logps/rejected": -378.17291259765625, |
| "loss": 0.5518, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.8876248598098755, |
| "rewards/margins": 0.491793155670166, |
| "rewards/rejected": -1.3794180154800415, |
| "step": 1430 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.882049274743578e-06, |
| "logits/chosen": -1.7279183864593506, |
| "logits/rejected": -1.3486309051513672, |
| "logps/chosen": -405.9081115722656, |
| "logps/rejected": -435.51708984375, |
| "loss": 0.5283, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.9554494619369507, |
| "rewards/margins": 0.64899742603302, |
| "rewards/rejected": -1.6044470071792603, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.878557331916729e-06, |
| "logits/chosen": -1.4362828731536865, |
| "logits/rejected": -1.2300403118133545, |
| "logps/chosen": -356.7294006347656, |
| "logps/rejected": -382.6611633300781, |
| "loss": 0.5344, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.9905643463134766, |
| "rewards/margins": 0.5484082698822021, |
| "rewards/rejected": -1.5389726161956787, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.875015736617576e-06, |
| "logits/chosen": -1.6814028024673462, |
| "logits/rejected": -1.2631704807281494, |
| "logps/chosen": -434.5621643066406, |
| "logps/rejected": -433.96038818359375, |
| "loss": 0.5972, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.0096790790557861, |
| "rewards/margins": 0.5042110681533813, |
| "rewards/rejected": -1.5138901472091675, |
| "step": 1460 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.8714245627770515e-06, |
| "logits/chosen": -1.7851835489273071, |
| "logits/rejected": -1.0083516836166382, |
| "logps/chosen": -348.3280334472656, |
| "logps/rejected": -359.25750732421875, |
| "loss": 0.6018, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -1.0166282653808594, |
| "rewards/margins": 0.45403042435646057, |
| "rewards/rejected": -1.470658540725708, |
| "step": 1470 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.8677838853610445e-06, |
| "logits/chosen": -1.6132957935333252, |
| "logits/rejected": -0.8410366177558899, |
| "logps/chosen": -358.55548095703125, |
| "logps/rejected": -369.1767883300781, |
| "loss": 0.5801, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.9697539210319519, |
| "rewards/margins": 0.5656896829605103, |
| "rewards/rejected": -1.535443663597107, |
| "step": 1480 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.864093780368828e-06, |
| "logits/chosen": -1.8534510135650635, |
| "logits/rejected": -1.0167924165725708, |
| "logps/chosen": -379.2197265625, |
| "logps/rejected": -396.7680358886719, |
| "loss": 0.4763, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.8626648783683777, |
| "rewards/margins": 0.758463978767395, |
| "rewards/rejected": -1.621128797531128, |
| "step": 1490 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.860354324831482e-06, |
| "logits/chosen": -1.2768698930740356, |
| "logits/rejected": -1.3293505907058716, |
| "logps/chosen": -382.0423583984375, |
| "logps/rejected": -450.2859802246094, |
| "loss": 0.5456, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.1258668899536133, |
| "rewards/margins": 0.6569615602493286, |
| "rewards/rejected": -1.7828283309936523, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.2, |
| "eval_logits/chosen": -0.15739573538303375, |
| "eval_logits/rejected": 0.3097783625125885, |
| "eval_logps/chosen": -386.943603515625, |
| "eval_logps/rejected": -427.7158203125, |
| "eval_loss": 0.5645538568496704, |
| "eval_rewards/accuracies": 0.703499972820282, |
| "eval_rewards/chosen": -1.1845359802246094, |
| "eval_rewards/margins": 0.606810986995697, |
| "eval_rewards/rejected": -1.7913470268249512, |
| "eval_runtime": 1155.3202, |
| "eval_samples_per_second": 1.731, |
| "eval_steps_per_second": 0.866, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.856565596810279e-06, |
| "logits/chosen": -1.3375440835952759, |
| "logits/rejected": -0.9983604550361633, |
| "logps/chosen": -339.1542053222656, |
| "logps/rejected": -410.56072998046875, |
| "loss": 0.5896, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.2330477237701416, |
| "rewards/margins": 0.5682455897331238, |
| "rewards/rejected": -1.8012933731079102, |
| "step": 1510 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.852727675395056e-06, |
| "logits/chosen": -1.191486120223999, |
| "logits/rejected": -0.736042857170105, |
| "logps/chosen": -383.180419921875, |
| "logps/rejected": -435.84185791015625, |
| "loss": 0.4452, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.2295153141021729, |
| "rewards/margins": 0.8655103445053101, |
| "rewards/rejected": -2.0950255393981934, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.848840640702565e-06, |
| "logits/chosen": -1.4446022510528564, |
| "logits/rejected": -1.0689058303833008, |
| "logps/chosen": -384.16009521484375, |
| "logps/rejected": -390.2742614746094, |
| "loss": 0.6654, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -1.3684113025665283, |
| "rewards/margins": 0.37485355138778687, |
| "rewards/rejected": -1.74326491355896, |
| "step": 1530 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.844904573874798e-06, |
| "logits/chosen": -1.4517594575881958, |
| "logits/rejected": -0.9329123497009277, |
| "logps/chosen": -404.48828125, |
| "logps/rejected": -426.7264709472656, |
| "loss": 0.5109, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.2724699974060059, |
| "rewards/margins": 0.7288385629653931, |
| "rewards/rejected": -2.0013086795806885, |
| "step": 1540 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.840919557077297e-06, |
| "logits/chosen": -1.6456127166748047, |
| "logits/rejected": -0.6936973333358765, |
| "logps/chosen": -370.9361267089844, |
| "logps/rejected": -393.5981140136719, |
| "loss": 0.5758, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.0427272319793701, |
| "rewards/margins": 0.5629565119743347, |
| "rewards/rejected": -1.60568368434906, |
| "step": 1550 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 4.836885673497435e-06, |
| "logits/chosen": -1.382566213607788, |
| "logits/rejected": -0.757800817489624, |
| "logps/chosen": -366.60137939453125, |
| "logps/rejected": -412.43865966796875, |
| "loss": 0.501, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.9314566850662231, |
| "rewards/margins": 0.710411548614502, |
| "rewards/rejected": -1.6418683528900146, |
| "step": 1560 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.832803007342679e-06, |
| "logits/chosen": -0.9425197839736938, |
| "logits/rejected": -0.9502668380737305, |
| "logps/chosen": -353.66204833984375, |
| "logps/rejected": -422.31640625, |
| "loss": 0.5845, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.1626675128936768, |
| "rewards/margins": 0.5805758237838745, |
| "rewards/rejected": -1.7432434558868408, |
| "step": 1570 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.828671643838839e-06, |
| "logits/chosen": -1.275815725326538, |
| "logits/rejected": -0.9463005065917969, |
| "logps/chosen": -385.7954406738281, |
| "logps/rejected": -382.41021728515625, |
| "loss": 0.6096, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.1785976886749268, |
| "rewards/margins": 0.4413912892341614, |
| "rewards/rejected": -1.619989037513733, |
| "step": 1580 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.824491669228279e-06, |
| "logits/chosen": -1.289780616760254, |
| "logits/rejected": -0.7804897427558899, |
| "logps/chosen": -371.3612060546875, |
| "logps/rejected": -400.07611083984375, |
| "loss": 0.5764, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.173409104347229, |
| "rewards/margins": 0.5362740159034729, |
| "rewards/rejected": -1.7096830606460571, |
| "step": 1590 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.8202631707681245e-06, |
| "logits/chosen": -1.3760565519332886, |
| "logits/rejected": -0.5771588683128357, |
| "logps/chosen": -360.80352783203125, |
| "logps/rejected": -426.24688720703125, |
| "loss": 0.4722, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.24233078956604, |
| "rewards/margins": 0.8159643411636353, |
| "rewards/rejected": -2.0582950115203857, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.21, |
| "eval_logits/chosen": 0.03460278362035751, |
| "eval_logits/rejected": 0.539508044719696, |
| "eval_logps/chosen": -400.91131591796875, |
| "eval_logps/rejected": -442.8173828125, |
| "eval_loss": 0.5598156452178955, |
| "eval_rewards/accuracies": 0.7074999809265137, |
| "eval_rewards/chosen": -1.3242131471633911, |
| "eval_rewards/margins": 0.6181491613388062, |
| "eval_rewards/rejected": -1.9423623085021973, |
| "eval_runtime": 1154.908, |
| "eval_samples_per_second": 1.732, |
| "eval_steps_per_second": 0.866, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.815986236728437e-06, |
| "logits/chosen": -1.1099474430084229, |
| "logits/rejected": -0.6507538557052612, |
| "logps/chosen": -378.3443298339844, |
| "logps/rejected": -439.36651611328125, |
| "loss": 0.5462, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.2413318157196045, |
| "rewards/margins": 0.6197353601455688, |
| "rewards/rejected": -1.8610671758651733, |
| "step": 1610 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.811660956390372e-06, |
| "logits/chosen": -1.168228030204773, |
| "logits/rejected": -0.9048255085945129, |
| "logps/chosen": -420.23974609375, |
| "logps/rejected": -447.28790283203125, |
| "loss": 0.5796, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.1689625978469849, |
| "rewards/margins": 0.5838147401809692, |
| "rewards/rejected": -1.7527774572372437, |
| "step": 1620 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.807287420044319e-06, |
| "logits/chosen": -1.4433248043060303, |
| "logits/rejected": -0.7626504302024841, |
| "logps/chosen": -344.18035888671875, |
| "logps/rejected": -398.12451171875, |
| "loss": 0.5509, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.1426336765289307, |
| "rewards/margins": 0.6661030650138855, |
| "rewards/rejected": -1.8087365627288818, |
| "step": 1630 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 4.802865718988008e-06, |
| "logits/chosen": -1.0445847511291504, |
| "logits/rejected": -1.0025596618652344, |
| "logps/chosen": -345.99066162109375, |
| "logps/rejected": -438.6319885253906, |
| "loss": 0.5689, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.2196751832962036, |
| "rewards/margins": 0.5887807607650757, |
| "rewards/rejected": -1.8084558248519897, |
| "step": 1640 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.798395945524615e-06, |
| "logits/chosen": -1.2105060815811157, |
| "logits/rejected": -0.5044958591461182, |
| "logps/chosen": -390.4164123535156, |
| "logps/rejected": -438.38916015625, |
| "loss": 0.5252, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.3014953136444092, |
| "rewards/margins": 0.7168537974357605, |
| "rewards/rejected": -2.0183491706848145, |
| "step": 1650 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.793878192960823e-06, |
| "logits/chosen": -1.5575644969940186, |
| "logits/rejected": -0.6008479595184326, |
| "logps/chosen": -447.37384033203125, |
| "logps/rejected": -512.7503051757812, |
| "loss": 0.5289, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.3483020067214966, |
| "rewards/margins": 0.7725515961647034, |
| "rewards/rejected": -2.1208536624908447, |
| "step": 1660 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.789312555604887e-06, |
| "logits/chosen": -1.3201217651367188, |
| "logits/rejected": -0.7254796028137207, |
| "logps/chosen": -357.54486083984375, |
| "logps/rejected": -404.8153381347656, |
| "loss": 0.5135, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.0712493658065796, |
| "rewards/margins": 0.709019660949707, |
| "rewards/rejected": -1.7802692651748657, |
| "step": 1670 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.784699128764654e-06, |
| "logits/chosen": -1.4174318313598633, |
| "logits/rejected": -0.4926614761352539, |
| "logps/chosen": -362.864013671875, |
| "logps/rejected": -409.38238525390625, |
| "loss": 0.5552, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.1419918537139893, |
| "rewards/margins": 0.6905233263969421, |
| "rewards/rejected": -1.8325151205062866, |
| "step": 1680 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.780038008745581e-06, |
| "logits/chosen": -1.2989250421524048, |
| "logits/rejected": -0.561491072177887, |
| "logps/chosen": -437.3382263183594, |
| "logps/rejected": -468.6178283691406, |
| "loss": 0.5451, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -1.4737846851348877, |
| "rewards/margins": 0.7077856063842773, |
| "rewards/rejected": -2.181570291519165, |
| "step": 1690 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.775329292848721e-06, |
| "logits/chosen": -0.8505582809448242, |
| "logits/rejected": -0.4519156515598297, |
| "logps/chosen": -432.04736328125, |
| "logps/rejected": -503.6212463378906, |
| "loss": 0.5072, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.4707634449005127, |
| "rewards/margins": 0.8545511960983276, |
| "rewards/rejected": -2.32531476020813, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.22, |
| "eval_logits/chosen": 0.46571579575538635, |
| "eval_logits/rejected": 1.0410724878311157, |
| "eval_logps/chosen": -418.8860168457031, |
| "eval_logps/rejected": -465.253662109375, |
| "eval_loss": 0.5574353933334351, |
| "eval_rewards/accuracies": 0.7059999704360962, |
| "eval_rewards/chosen": -1.503960132598877, |
| "eval_rewards/margins": 0.6627644896507263, |
| "eval_rewards/rejected": -2.166724681854248, |
| "eval_runtime": 1172.1157, |
| "eval_samples_per_second": 1.706, |
| "eval_steps_per_second": 0.853, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 4.770573079368691e-06, |
| "logits/chosen": -0.927274227142334, |
| "logits/rejected": -0.7411154508590698, |
| "logps/chosen": -412.53338623046875, |
| "logps/rejected": -437.78875732421875, |
| "loss": 0.6122, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -1.4526969194412231, |
| "rewards/margins": 0.5428808927536011, |
| "rewards/rejected": -1.9955780506134033, |
| "step": 1710 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.765769467591626e-06, |
| "logits/chosen": -1.1742867231369019, |
| "logits/rejected": -0.6260591745376587, |
| "logps/chosen": -435.9891052246094, |
| "logps/rejected": -470.205810546875, |
| "loss": 0.5436, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.4501960277557373, |
| "rewards/margins": 0.603704035282135, |
| "rewards/rejected": -2.0539000034332275, |
| "step": 1720 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.760918557793096e-06, |
| "logits/chosen": -1.0430524349212646, |
| "logits/rejected": -0.6410683989524841, |
| "logps/chosen": -408.3353271484375, |
| "logps/rejected": -473.285888671875, |
| "loss": 0.5752, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.56908118724823, |
| "rewards/margins": 0.5391335487365723, |
| "rewards/rejected": -2.108214855194092, |
| "step": 1730 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.756020451236025e-06, |
| "logits/chosen": -1.3290060758590698, |
| "logits/rejected": -0.5599908232688904, |
| "logps/chosen": -448.45428466796875, |
| "logps/rejected": -473.66387939453125, |
| "loss": 0.5918, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.4412941932678223, |
| "rewards/margins": 0.5439326167106628, |
| "rewards/rejected": -1.9852268695831299, |
| "step": 1740 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.751075250168569e-06, |
| "logits/chosen": -1.4968469142913818, |
| "logits/rejected": 0.1215222105383873, |
| "logps/chosen": -433.72222900390625, |
| "logps/rejected": -473.5547790527344, |
| "loss": 0.539, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.6900333166122437, |
| "rewards/margins": 0.7276581525802612, |
| "rewards/rejected": -2.417691707611084, |
| "step": 1750 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.746083057821981e-06, |
| "logits/chosen": -1.1097086668014526, |
| "logits/rejected": -0.21016299724578857, |
| "logps/chosen": -395.05413818359375, |
| "logps/rejected": -453.3968811035156, |
| "loss": 0.5255, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.454843282699585, |
| "rewards/margins": 0.9116643667221069, |
| "rewards/rejected": -2.3665075302124023, |
| "step": 1760 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.741043978408463e-06, |
| "logits/chosen": -0.9275191426277161, |
| "logits/rejected": -0.33824652433395386, |
| "logps/chosen": -380.26177978515625, |
| "logps/rejected": -466.066650390625, |
| "loss": 0.509, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.2040696144104004, |
| "rewards/margins": 0.8978725671768188, |
| "rewards/rejected": -2.1019420623779297, |
| "step": 1770 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.735958117118983e-06, |
| "logits/chosen": -1.4339239597320557, |
| "logits/rejected": -0.27769067883491516, |
| "logps/chosen": -403.5133056640625, |
| "logps/rejected": -451.6259765625, |
| "loss": 0.5387, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.0971940755844116, |
| "rewards/margins": 0.7503548860549927, |
| "rewards/rejected": -1.8475488424301147, |
| "step": 1780 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 4.730825580121084e-06, |
| "logits/chosen": -1.1844618320465088, |
| "logits/rejected": -0.35769081115722656, |
| "logps/chosen": -366.8836669921875, |
| "logps/rejected": -446.3826599121094, |
| "loss": 0.5439, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.2104787826538086, |
| "rewards/margins": 0.7347462177276611, |
| "rewards/rejected": -1.9452251195907593, |
| "step": 1790 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.725646474556666e-06, |
| "logits/chosen": -0.8105975985527039, |
| "logits/rejected": -0.6895971894264221, |
| "logps/chosen": -352.57110595703125, |
| "logps/rejected": -445.0667419433594, |
| "loss": 0.5284, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.290921926498413, |
| "rewards/margins": 0.7879074811935425, |
| "rewards/rejected": -2.078829526901245, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.24, |
| "eval_logits/chosen": 0.6528418064117432, |
| "eval_logits/rejected": 1.2403807640075684, |
| "eval_logps/chosen": -423.3541564941406, |
| "eval_logps/rejected": -469.1292724609375, |
| "eval_loss": 0.5534334182739258, |
| "eval_rewards/accuracies": 0.7070000171661377, |
| "eval_rewards/chosen": -1.5486416816711426, |
| "eval_rewards/margins": 0.6568393111228943, |
| "eval_rewards/rejected": -2.2054810523986816, |
| "eval_runtime": 1170.7626, |
| "eval_samples_per_second": 1.708, |
| "eval_steps_per_second": 0.854, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.720420908539748e-06, |
| "logits/chosen": -1.189819574356079, |
| "logits/rejected": -0.6282674074172974, |
| "logps/chosen": -418.718505859375, |
| "logps/rejected": -480.90216064453125, |
| "loss": 0.6174, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.7022778987884521, |
| "rewards/margins": 0.5764120817184448, |
| "rewards/rejected": -2.2786898612976074, |
| "step": 1810 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.715148991154216e-06, |
| "logits/chosen": -1.222535252571106, |
| "logits/rejected": -0.8831266164779663, |
| "logps/chosen": -516.7198486328125, |
| "logps/rejected": -564.3641967773438, |
| "loss": 0.586, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7563714981079102, |
| "rewards/margins": 0.5409680604934692, |
| "rewards/rejected": -2.297339677810669, |
| "step": 1820 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.709830832451538e-06, |
| "logits/chosen": -0.8930926322937012, |
| "logits/rejected": -0.4406977593898773, |
| "logps/chosen": -486.24578857421875, |
| "logps/rejected": -537.7103881835938, |
| "loss": 0.5226, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.8711732625961304, |
| "rewards/margins": 0.6819201111793518, |
| "rewards/rejected": -2.553093433380127, |
| "step": 1830 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.704466543448477e-06, |
| "logits/chosen": -1.0724461078643799, |
| "logits/rejected": -0.03855453059077263, |
| "logps/chosen": -519.872802734375, |
| "logps/rejected": -553.6289672851562, |
| "loss": 0.4735, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.7257235050201416, |
| "rewards/margins": 0.9186381101608276, |
| "rewards/rejected": -2.644361972808838, |
| "step": 1840 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.699056236124762e-06, |
| "logits/chosen": -1.1924560070037842, |
| "logits/rejected": -0.6589155197143555, |
| "logps/chosen": -442.31707763671875, |
| "logps/rejected": -505.4425354003906, |
| "loss": 0.6075, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.8461265563964844, |
| "rewards/margins": 0.616051197052002, |
| "rewards/rejected": -2.4621777534484863, |
| "step": 1850 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.693600023420758e-06, |
| "logits/chosen": -1.2352359294891357, |
| "logits/rejected": -0.2970736026763916, |
| "logps/chosen": -436.0951232910156, |
| "logps/rejected": -452.52923583984375, |
| "loss": 0.4406, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.391472339630127, |
| "rewards/margins": 0.9484335780143738, |
| "rewards/rejected": -2.3399059772491455, |
| "step": 1860 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 4.688098019235108e-06, |
| "logits/chosen": -1.2450083494186401, |
| "logits/rejected": -0.5829068422317505, |
| "logps/chosen": -445.16143798828125, |
| "logps/rejected": -517.4144287109375, |
| "loss": 0.5586, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.5391108989715576, |
| "rewards/margins": 0.8366822004318237, |
| "rewards/rejected": -2.375793218612671, |
| "step": 1870 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.682550338422353e-06, |
| "logits/chosen": -1.3568122386932373, |
| "logits/rejected": -0.5020118355751038, |
| "logps/chosen": -410.96826171875, |
| "logps/rejected": -465.6785583496094, |
| "loss": 0.4733, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.4551056623458862, |
| "rewards/margins": 0.9018338322639465, |
| "rewards/rejected": -2.3569395542144775, |
| "step": 1880 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.676957096790536e-06, |
| "logits/chosen": -1.0935922861099243, |
| "logits/rejected": -0.5090438723564148, |
| "logps/chosen": -427.3106994628906, |
| "logps/rejected": -443.31231689453125, |
| "loss": 0.6429, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.6104503870010376, |
| "rewards/margins": 0.5206896066665649, |
| "rewards/rejected": -2.1311399936676025, |
| "step": 1890 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.671318411098782e-06, |
| "logits/chosen": -0.7024677991867065, |
| "logits/rejected": -0.8284416198730469, |
| "logps/chosen": -458.29608154296875, |
| "logps/rejected": -553.300537109375, |
| "loss": 0.5623, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.7181098461151123, |
| "rewards/margins": 0.9186338186264038, |
| "rewards/rejected": -2.6367437839508057, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.25, |
| "eval_logits/chosen": 0.30577704310417175, |
| "eval_logits/rejected": 0.7807530760765076, |
| "eval_logps/chosen": -439.5539245605469, |
| "eval_logps/rejected": -491.0526123046875, |
| "eval_loss": 0.5624514222145081, |
| "eval_rewards/accuracies": 0.7055000066757202, |
| "eval_rewards/chosen": -1.7106391191482544, |
| "eval_rewards/margins": 0.7140753269195557, |
| "eval_rewards/rejected": -2.4247145652770996, |
| "eval_runtime": 1175.0312, |
| "eval_samples_per_second": 1.702, |
| "eval_steps_per_second": 0.851, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.665634399054864e-06, |
| "logits/chosen": -0.725782036781311, |
| "logits/rejected": -0.6188144683837891, |
| "logps/chosen": -409.5749206542969, |
| "logps/rejected": -466.3291931152344, |
| "loss": 0.663, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.785436987876892, |
| "rewards/margins": 0.5989683270454407, |
| "rewards/rejected": -2.3844053745269775, |
| "step": 1910 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.659905179312743e-06, |
| "logits/chosen": -1.4557887315750122, |
| "logits/rejected": -0.6816984415054321, |
| "logps/chosen": -442.3401794433594, |
| "logps/rejected": -428.6465759277344, |
| "loss": 0.6225, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.4092276096343994, |
| "rewards/margins": 0.5179110765457153, |
| "rewards/rejected": -1.9271386861801147, |
| "step": 1920 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.654130871470093e-06, |
| "logits/chosen": -1.4024882316589355, |
| "logits/rejected": -0.5982595682144165, |
| "logps/chosen": -372.2467346191406, |
| "logps/rejected": -375.06610107421875, |
| "loss": 0.6068, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.1297688484191895, |
| "rewards/margins": 0.4118806719779968, |
| "rewards/rejected": -1.541649580001831, |
| "step": 1930 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.6483115960658045e-06, |
| "logits/chosen": -1.4664485454559326, |
| "logits/rejected": -0.3487986624240875, |
| "logps/chosen": -396.49334716796875, |
| "logps/rejected": -393.38616943359375, |
| "loss": 0.5078, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.2406772375106812, |
| "rewards/margins": 0.653410792350769, |
| "rewards/rejected": -1.8940880298614502, |
| "step": 1940 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.642447474577466e-06, |
| "logits/chosen": -1.0883629322052002, |
| "logits/rejected": -1.072458028793335, |
| "logps/chosen": -371.15093994140625, |
| "logps/rejected": -431.68426513671875, |
| "loss": 0.5129, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.395630121231079, |
| "rewards/margins": 0.68388831615448, |
| "rewards/rejected": -2.0795183181762695, |
| "step": 1950 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.636538629418832e-06, |
| "logits/chosen": -1.2723660469055176, |
| "logits/rejected": -0.904948353767395, |
| "logps/chosen": -401.46112060546875, |
| "logps/rejected": -466.59393310546875, |
| "loss": 0.4758, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.1891543865203857, |
| "rewards/margins": 0.8602797389030457, |
| "rewards/rejected": -2.049434185028076, |
| "step": 1960 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.630585183937263e-06, |
| "logits/chosen": -1.2972795963287354, |
| "logits/rejected": -0.6552487015724182, |
| "logps/chosen": -409.4198913574219, |
| "logps/rejected": -429.36248779296875, |
| "loss": 0.5831, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.228527307510376, |
| "rewards/margins": 0.5128410458564758, |
| "rewards/rejected": -1.741368055343628, |
| "step": 1970 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.6245872624111535e-06, |
| "logits/chosen": -1.1344765424728394, |
| "logits/rejected": -0.9775272607803345, |
| "logps/chosen": -344.75238037109375, |
| "logps/rejected": -382.8861389160156, |
| "loss": 0.615, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.2352514266967773, |
| "rewards/margins": 0.49147820472717285, |
| "rewards/rejected": -1.7267297506332397, |
| "step": 1980 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.618544990047336e-06, |
| "logits/chosen": -1.1246018409729004, |
| "logits/rejected": -0.5179897546768188, |
| "logps/chosen": -425.73406982421875, |
| "logps/rejected": -470.3760681152344, |
| "loss": 0.5957, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.2693254947662354, |
| "rewards/margins": 0.6230798959732056, |
| "rewards/rejected": -1.8924052715301514, |
| "step": 1990 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.612458492978473e-06, |
| "logits/chosen": -1.3398029804229736, |
| "logits/rejected": -0.8593130111694336, |
| "logps/chosen": -380.1425476074219, |
| "logps/rejected": -439.82891845703125, |
| "loss": 0.6092, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.3958600759506226, |
| "rewards/margins": 0.515204131603241, |
| "rewards/rejected": -1.9110643863677979, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.26, |
| "eval_logits/chosen": 0.007945289835333824, |
| "eval_logits/rejected": 0.519872784614563, |
| "eval_logps/chosen": -370.0728454589844, |
| "eval_logps/rejected": -413.70892333984375, |
| "eval_loss": 0.550082266330719, |
| "eval_rewards/accuracies": 0.7085000276565552, |
| "eval_rewards/chosen": -1.0158281326293945, |
| "eval_rewards/margins": 0.6354495882987976, |
| "eval_rewards/rejected": -1.6512778997421265, |
| "eval_runtime": 1182.5268, |
| "eval_samples_per_second": 1.691, |
| "eval_steps_per_second": 0.846, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.606327898260413e-06, |
| "logits/chosen": -1.1447194814682007, |
| "logits/rejected": -0.6674858331680298, |
| "logps/chosen": -386.634765625, |
| "logps/rejected": -428.674072265625, |
| "loss": 0.5888, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.0479754209518433, |
| "rewards/margins": 0.6669696569442749, |
| "rewards/rejected": -1.7149450778961182, |
| "step": 2010 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 4.600153333869549e-06, |
| "logits/chosen": -1.5867061614990234, |
| "logits/rejected": -0.8068370819091797, |
| "logps/chosen": -372.733154296875, |
| "logps/rejected": -399.0820007324219, |
| "loss": 0.5192, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.9379202127456665, |
| "rewards/margins": 0.6650333404541016, |
| "rewards/rejected": -1.602953553199768, |
| "step": 2020 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.593934928700141e-06, |
| "logits/chosen": -1.5054073333740234, |
| "logits/rejected": -0.331088125705719, |
| "logps/chosen": -396.2590026855469, |
| "logps/rejected": -440.35308837890625, |
| "loss": 0.516, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.2215917110443115, |
| "rewards/margins": 0.7985371947288513, |
| "rewards/rejected": -2.0201287269592285, |
| "step": 2030 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.587672812561626e-06, |
| "logits/chosen": -1.0228203535079956, |
| "logits/rejected": -0.6623596549034119, |
| "logps/chosen": -359.9720458984375, |
| "logps/rejected": -467.29461669921875, |
| "loss": 0.5241, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.2281525135040283, |
| "rewards/margins": 0.7495523691177368, |
| "rewards/rejected": -1.9777047634124756, |
| "step": 2040 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.581367116175911e-06, |
| "logits/chosen": -0.9328869581222534, |
| "logits/rejected": -0.2950093150138855, |
| "logps/chosen": -431.26873779296875, |
| "logps/rejected": -445.1465759277344, |
| "loss": 0.5993, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.3680379390716553, |
| "rewards/margins": 0.5561319589614868, |
| "rewards/rejected": -1.9241701364517212, |
| "step": 2050 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.5750179711746416e-06, |
| "logits/chosen": -0.9565197229385376, |
| "logits/rejected": -0.35139861702919006, |
| "logps/chosen": -391.14947509765625, |
| "logps/rejected": -434.99713134765625, |
| "loss": 0.5767, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.3655421733856201, |
| "rewards/margins": 0.5043060183525085, |
| "rewards/rejected": -1.8698484897613525, |
| "step": 2060 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.5686255100964535e-06, |
| "logits/chosen": -1.3030929565429688, |
| "logits/rejected": -0.6504136323928833, |
| "logps/chosen": -408.542236328125, |
| "logps/rejected": -433.14068603515625, |
| "loss": 0.5608, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.455055832862854, |
| "rewards/margins": 0.6146378517150879, |
| "rewards/rejected": -2.0696938037872314, |
| "step": 2070 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.562189866384209e-06, |
| "logits/chosen": -0.8565002679824829, |
| "logits/rejected": -0.5893241763114929, |
| "logps/chosen": -393.67462158203125, |
| "logps/rejected": -500.3961486816406, |
| "loss": 0.5081, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.5734975337982178, |
| "rewards/margins": 0.8559523820877075, |
| "rewards/rejected": -2.429450035095215, |
| "step": 2080 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.555711174382209e-06, |
| "logits/chosen": -1.1275274753570557, |
| "logits/rejected": -0.4757865369319916, |
| "logps/chosen": -406.5447692871094, |
| "logps/rejected": -458.0465393066406, |
| "loss": 0.5529, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7623049020767212, |
| "rewards/margins": 0.6915754079818726, |
| "rewards/rejected": -2.4538803100585938, |
| "step": 2090 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 4.549189569333387e-06, |
| "logits/chosen": -1.1001176834106445, |
| "logits/rejected": -0.4274144172668457, |
| "logps/chosen": -383.9236755371094, |
| "logps/rejected": -410.41162109375, |
| "loss": 0.5726, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.4652644395828247, |
| "rewards/margins": 0.5952554941177368, |
| "rewards/rejected": -2.0605199337005615, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.27, |
| "eval_logits/chosen": 0.4405115842819214, |
| "eval_logits/rejected": 0.9980767965316772, |
| "eval_logps/chosen": -415.4569396972656, |
| "eval_logps/rejected": -464.3841857910156, |
| "eval_loss": 0.5433006882667542, |
| "eval_rewards/accuracies": 0.7149999737739563, |
| "eval_rewards/chosen": -1.4696691036224365, |
| "eval_rewards/margins": 0.6883615255355835, |
| "eval_rewards/rejected": -2.1580307483673096, |
| "eval_runtime": 1178.0191, |
| "eval_samples_per_second": 1.698, |
| "eval_steps_per_second": 0.849, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.542625187376491e-06, |
| "logits/chosen": -1.145003318786621, |
| "logits/rejected": 0.003861379576846957, |
| "logps/chosen": -429.87255859375, |
| "logps/rejected": -454.26239013671875, |
| "loss": 0.5476, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.3411312103271484, |
| "rewards/margins": 0.6698344349861145, |
| "rewards/rejected": -2.0109658241271973, |
| "step": 2110 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.536018165543239e-06, |
| "logits/chosen": -0.990827739238739, |
| "logits/rejected": -0.4457016885280609, |
| "logps/chosen": -453.3412170410156, |
| "logps/rejected": -506.03741455078125, |
| "loss": 0.5634, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.5011839866638184, |
| "rewards/margins": 0.6426320672035217, |
| "rewards/rejected": -2.1438162326812744, |
| "step": 2120 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.529368641755453e-06, |
| "logits/chosen": -0.9683168530464172, |
| "logits/rejected": -0.37535277009010315, |
| "logps/chosen": -391.17059326171875, |
| "logps/rejected": -449.50714111328125, |
| "loss": 0.6002, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.6766704320907593, |
| "rewards/margins": 0.6462152600288391, |
| "rewards/rejected": -2.322885513305664, |
| "step": 2130 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.522676754822189e-06, |
| "logits/chosen": -1.1304986476898193, |
| "logits/rejected": -0.19757482409477234, |
| "logps/chosen": -427.12188720703125, |
| "logps/rejected": -439.59716796875, |
| "loss": 0.4706, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.5254873037338257, |
| "rewards/margins": 0.839495837688446, |
| "rewards/rejected": -2.364983081817627, |
| "step": 2140 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.515942644436836e-06, |
| "logits/chosen": -1.0416542291641235, |
| "logits/rejected": -0.08233954012393951, |
| "logps/chosen": -426.8787536621094, |
| "logps/rejected": -478.9959411621094, |
| "loss": 0.5463, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -1.4865224361419678, |
| "rewards/margins": 0.7738333940505981, |
| "rewards/rejected": -2.2603557109832764, |
| "step": 2150 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.509166451174194e-06, |
| "logits/chosen": -1.0556268692016602, |
| "logits/rejected": -0.6550186276435852, |
| "logps/chosen": -412.85565185546875, |
| "logps/rejected": -469.2373962402344, |
| "loss": 0.5133, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.0766072273254395, |
| "rewards/margins": 0.8188613057136536, |
| "rewards/rejected": -1.8954684734344482, |
| "step": 2160 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 4.502348316487552e-06, |
| "logits/chosen": -1.5349935293197632, |
| "logits/rejected": -0.5842410922050476, |
| "logps/chosen": -392.6039123535156, |
| "logps/rejected": -432.892578125, |
| "loss": 0.5352, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.1731538772583008, |
| "rewards/margins": 0.7138124704360962, |
| "rewards/rejected": -1.886966347694397, |
| "step": 2170 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.495488382705722e-06, |
| "logits/chosen": -1.4572819471359253, |
| "logits/rejected": -0.1267738789319992, |
| "logps/chosen": -435.73748779296875, |
| "logps/rejected": -435.91864013671875, |
| "loss": 0.4479, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.9534079432487488, |
| "rewards/margins": 0.9101373553276062, |
| "rewards/rejected": -1.8635451793670654, |
| "step": 2180 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.488586793030075e-06, |
| "logits/chosen": -1.0830357074737549, |
| "logits/rejected": -0.5002374053001404, |
| "logps/chosen": -330.9154052734375, |
| "logps/rejected": -439.44158935546875, |
| "loss": 0.4391, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.0812445878982544, |
| "rewards/margins": 0.9342634081840515, |
| "rewards/rejected": -2.015507936477661, |
| "step": 2190 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.481643691531551e-06, |
| "logits/chosen": -1.0210946798324585, |
| "logits/rejected": -0.11824611574411392, |
| "logps/chosen": -396.3450622558594, |
| "logps/rejected": -435.77734375, |
| "loss": 0.5323, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.2495872974395752, |
| "rewards/margins": 0.7661415934562683, |
| "rewards/rejected": -2.015728712081909, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.29, |
| "eval_logits/chosen": 0.7444968819618225, |
| "eval_logits/rejected": 1.3532707691192627, |
| "eval_logps/chosen": -400.2243957519531, |
| "eval_logps/rejected": -457.445068359375, |
| "eval_loss": 0.5483363270759583, |
| "eval_rewards/accuracies": 0.7149999737739563, |
| "eval_rewards/chosen": -1.3173435926437378, |
| "eval_rewards/margins": 0.7712955474853516, |
| "eval_rewards/rejected": -2.088639259338379, |
| "eval_runtime": 1176.3943, |
| "eval_samples_per_second": 1.7, |
| "eval_steps_per_second": 0.85, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.474659223147652e-06, |
| "logits/chosen": -0.6220839619636536, |
| "logits/rejected": -0.34505695104599, |
| "logps/chosen": -400.8376770019531, |
| "logps/rejected": -455.2310485839844, |
| "loss": 0.5745, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.3072404861450195, |
| "rewards/margins": 0.7971282005310059, |
| "rewards/rejected": -2.1043686866760254, |
| "step": 2210 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.4676335336794125e-06, |
| "logits/chosen": -0.9507938623428345, |
| "logits/rejected": -0.2239619791507721, |
| "logps/chosen": -421.60479736328125, |
| "logps/rejected": -466.75341796875, |
| "loss": 0.5746, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.1340086460113525, |
| "rewards/margins": 0.7016123533248901, |
| "rewards/rejected": -1.8356211185455322, |
| "step": 2220 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.46056676978836e-06, |
| "logits/chosen": -0.8311563730239868, |
| "logits/rejected": -0.7106046676635742, |
| "logps/chosen": -368.59271240234375, |
| "logps/rejected": -457.5370178222656, |
| "loss": 0.5991, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.149174451828003, |
| "rewards/margins": 0.5227919816970825, |
| "rewards/rejected": -1.671966314315796, |
| "step": 2230 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.453459078993453e-06, |
| "logits/chosen": -0.7870109677314758, |
| "logits/rejected": -0.6436534523963928, |
| "logps/chosen": -353.3741760253906, |
| "logps/rejected": -449.02093505859375, |
| "loss": 0.4118, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.9842308163642883, |
| "rewards/margins": 1.0269984006881714, |
| "rewards/rejected": -2.0112290382385254, |
| "step": 2240 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 4.446310609668001e-06, |
| "logits/chosen": -0.685552716255188, |
| "logits/rejected": -0.2860308289527893, |
| "logps/chosen": -392.5391540527344, |
| "logps/rejected": -509.0357360839844, |
| "loss": 0.5367, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.616072654724121, |
| "rewards/margins": 0.8003999590873718, |
| "rewards/rejected": -2.4164726734161377, |
| "step": 2250 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.439121511036562e-06, |
| "logits/chosen": -0.8480124473571777, |
| "logits/rejected": 0.001265025115571916, |
| "logps/chosen": -474.6297912597656, |
| "logps/rejected": -509.7176818847656, |
| "loss": 0.5317, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.8826634883880615, |
| "rewards/margins": 0.7242499589920044, |
| "rewards/rejected": -2.6069135665893555, |
| "step": 2260 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.431891933171839e-06, |
| "logits/chosen": -1.1753742694854736, |
| "logits/rejected": -0.3657965064048767, |
| "logps/chosen": -403.2115783691406, |
| "logps/rejected": -454.543212890625, |
| "loss": 0.6065, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -1.4112242460250854, |
| "rewards/margins": 0.6532294154167175, |
| "rewards/rejected": -2.064453601837158, |
| "step": 2270 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.424622026991536e-06, |
| "logits/chosen": -1.0060756206512451, |
| "logits/rejected": -0.37251150608062744, |
| "logps/chosen": -391.8915100097656, |
| "logps/rejected": -437.4137268066406, |
| "loss": 0.5717, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.2653111219406128, |
| "rewards/margins": 0.6680868864059448, |
| "rewards/rejected": -1.9333980083465576, |
| "step": 2280 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.417311944255215e-06, |
| "logits/chosen": -0.7842764258384705, |
| "logits/rejected": -0.8166142702102661, |
| "logps/chosen": -354.4258728027344, |
| "logps/rejected": -414.52362060546875, |
| "loss": 0.6835, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -1.1270352602005005, |
| "rewards/margins": 0.41290101408958435, |
| "rewards/rejected": -1.5399363040924072, |
| "step": 2290 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.409961837561122e-06, |
| "logits/chosen": -0.6530027985572815, |
| "logits/rejected": -0.5591267943382263, |
| "logps/chosen": -439.7125549316406, |
| "logps/rejected": -520.6340942382812, |
| "loss": 0.5148, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.2820568084716797, |
| "rewards/margins": 0.7924367785453796, |
| "rewards/rejected": -2.074493408203125, |
| "step": 2300 |
| }, |
| { |
| "epoch": 0.3, |
| "eval_logits/chosen": 0.5106939077377319, |
| "eval_logits/rejected": 1.1454416513442993, |
| "eval_logps/chosen": -400.4307861328125, |
| "eval_logps/rejected": -450.4645690917969, |
| "eval_loss": 0.5387266278266907, |
| "eval_rewards/accuracies": 0.7275000214576721, |
| "eval_rewards/chosen": -1.3194077014923096, |
| "eval_rewards/margins": 0.6994263529777527, |
| "eval_rewards/rejected": -2.018834114074707, |
| "eval_runtime": 1973.7027, |
| "eval_samples_per_second": 1.013, |
| "eval_steps_per_second": 0.507, |
| "step": 2300 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.402571860343006e-06, |
| "logits/chosen": -1.4138994216918945, |
| "logits/rejected": 0.06842954456806183, |
| "logps/chosen": -422.7046813964844, |
| "logps/rejected": -416.2068786621094, |
| "loss": 0.5745, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.4042686223983765, |
| "rewards/margins": 0.6060650944709778, |
| "rewards/rejected": -2.01033353805542, |
| "step": 2310 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.3951421668669165e-06, |
| "logits/chosen": -0.9971601366996765, |
| "logits/rejected": -0.3194156587123871, |
| "logps/chosen": -423.9295349121094, |
| "logps/rejected": -490.6680603027344, |
| "loss": 0.5077, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.4917466640472412, |
| "rewards/margins": 0.8139832615852356, |
| "rewards/rejected": -2.305730104446411, |
| "step": 2320 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 4.3876729122279784e-06, |
| "logits/chosen": -0.9761942028999329, |
| "logits/rejected": -0.7307599782943726, |
| "logps/chosen": -324.6410827636719, |
| "logps/rejected": -422.3362731933594, |
| "loss": 0.4951, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.2676632404327393, |
| "rewards/margins": 0.9440364837646484, |
| "rewards/rejected": -2.2116997241973877, |
| "step": 2330 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.3801642523471585e-06, |
| "logits/chosen": -1.422680139541626, |
| "logits/rejected": 0.22055339813232422, |
| "logps/chosen": -427.3160705566406, |
| "logps/rejected": -468.10980224609375, |
| "loss": 0.4991, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.5216461420059204, |
| "rewards/margins": 0.8185620307922363, |
| "rewards/rejected": -2.3402082920074463, |
| "step": 2340 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.37261634396801e-06, |
| "logits/chosen": -0.7539848685264587, |
| "logits/rejected": -0.37924566864967346, |
| "logps/chosen": -433.6114807128906, |
| "logps/rejected": -493.5924377441406, |
| "loss": 0.5182, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7175815105438232, |
| "rewards/margins": 0.7918481230735779, |
| "rewards/rejected": -2.509429693222046, |
| "step": 2350 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.365029344653401e-06, |
| "logits/chosen": -1.1154688596725464, |
| "logits/rejected": -0.17494897544384003, |
| "logps/chosen": -479.53936767578125, |
| "logps/rejected": -493.6295471191406, |
| "loss": 0.5254, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.4410346746444702, |
| "rewards/margins": 0.9099518656730652, |
| "rewards/rejected": -2.3509867191314697, |
| "step": 2360 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.35740341278222e-06, |
| "logits/chosen": -1.2674121856689453, |
| "logits/rejected": -0.6544126272201538, |
| "logps/chosen": -448.0379943847656, |
| "logps/rejected": -505.9681701660156, |
| "loss": 0.5445, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.2304657697677612, |
| "rewards/margins": 0.7070282697677612, |
| "rewards/rejected": -1.9374940395355225, |
| "step": 2370 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.349738707546079e-06, |
| "logits/chosen": -0.8608369827270508, |
| "logits/rejected": -0.4290972650051117, |
| "logps/chosen": -400.2996826171875, |
| "logps/rejected": -436.1752014160156, |
| "loss": 0.5362, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.3461062908172607, |
| "rewards/margins": 0.770219624042511, |
| "rewards/rejected": -2.116325855255127, |
| "step": 2380 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.3420353889459835e-06, |
| "logits/chosen": -1.3208414316177368, |
| "logits/rejected": -0.2934853434562683, |
| "logps/chosen": -462.20465087890625, |
| "logps/rejected": -489.04156494140625, |
| "loss": 0.4917, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.4787876605987549, |
| "rewards/margins": 0.8698205947875977, |
| "rewards/rejected": -2.3486080169677734, |
| "step": 2390 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.334293617788992e-06, |
| "logits/chosen": -1.3569272756576538, |
| "logits/rejected": 0.16524724662303925, |
| "logps/chosen": -401.202880859375, |
| "logps/rejected": -444.6561584472656, |
| "loss": 0.4112, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.5012043714523315, |
| "rewards/margins": 1.1782524585723877, |
| "rewards/rejected": -2.679456949234009, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.31, |
| "eval_logits/chosen": 0.6648316383361816, |
| "eval_logits/rejected": 1.2865513563156128, |
| "eval_logps/chosen": -430.5039978027344, |
| "eval_logps/rejected": -490.7723388671875, |
| "eval_loss": 0.5400993824005127, |
| "eval_rewards/accuracies": 0.7200000286102295, |
| "eval_rewards/chosen": -1.6201398372650146, |
| "eval_rewards/margins": 0.8017721772193909, |
| "eval_rewards/rejected": -2.4219119548797607, |
| "eval_runtime": 2327.0644, |
| "eval_samples_per_second": 0.859, |
| "eval_steps_per_second": 0.43, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.326513555684867e-06, |
| "logits/chosen": -1.254010558128357, |
| "logits/rejected": 0.03536475822329521, |
| "logps/chosen": -450.07354736328125, |
| "logps/rejected": -438.840576171875, |
| "loss": 0.6208, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.5773955583572388, |
| "rewards/margins": 0.5903611779212952, |
| "rewards/rejected": -2.1677565574645996, |
| "step": 2410 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.31869536504269e-06, |
| "logits/chosen": -0.6679142117500305, |
| "logits/rejected": -0.44401970505714417, |
| "logps/chosen": -402.5331115722656, |
| "logps/rejected": -454.895263671875, |
| "loss": 0.5753, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.4515701532363892, |
| "rewards/margins": 0.6856138110160828, |
| "rewards/rejected": -2.1371841430664062, |
| "step": 2420 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.310839209067482e-06, |
| "logits/chosen": -1.3650258779525757, |
| "logits/rejected": 0.0447537787258625, |
| "logps/chosen": -433.0634765625, |
| "logps/rejected": -456.85467529296875, |
| "loss": 0.5657, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.5849604606628418, |
| "rewards/margins": 0.5240734815597534, |
| "rewards/rejected": -2.1090340614318848, |
| "step": 2430 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.302945251756788e-06, |
| "logits/chosen": -0.8697819709777832, |
| "logits/rejected": -0.31318679451942444, |
| "logps/chosen": -399.88421630859375, |
| "logps/rejected": -466.7974548339844, |
| "loss": 0.4323, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.4232676029205322, |
| "rewards/margins": 0.9615268707275391, |
| "rewards/rejected": -2.3847947120666504, |
| "step": 2440 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.29501365789726e-06, |
| "logits/chosen": -0.6540710926055908, |
| "logits/rejected": -0.13195696473121643, |
| "logps/chosen": -375.75726318359375, |
| "logps/rejected": -441.58135986328125, |
| "loss": 0.5364, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.5193512439727783, |
| "rewards/margins": 0.8794568181037903, |
| "rewards/rejected": -2.398808002471924, |
| "step": 2450 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.2870445930612135e-06, |
| "logits/chosen": -0.6666491627693176, |
| "logits/rejected": -0.20681039988994598, |
| "logps/chosen": -443.6128845214844, |
| "logps/rejected": -499.73370361328125, |
| "loss": 0.4717, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.2950096130371094, |
| "rewards/margins": 1.0014543533325195, |
| "rewards/rejected": -2.296464204788208, |
| "step": 2460 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.279038223603171e-06, |
| "logits/chosen": -0.9633728265762329, |
| "logits/rejected": 0.12687508761882782, |
| "logps/chosen": -392.9165344238281, |
| "logps/rejected": -445.36907958984375, |
| "loss": 0.5094, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.3179303407669067, |
| "rewards/margins": 0.8730724453926086, |
| "rewards/rejected": -2.19100284576416, |
| "step": 2470 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 4.2709947166563906e-06, |
| "logits/chosen": -0.3830980658531189, |
| "logits/rejected": -0.04551839455962181, |
| "logps/chosen": -437.1377868652344, |
| "logps/rejected": -540.1201171875, |
| "loss": 0.5012, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7204186916351318, |
| "rewards/margins": 0.9347385168075562, |
| "rewards/rejected": -2.6551575660705566, |
| "step": 2480 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.262914240129379e-06, |
| "logits/chosen": -0.7016464471817017, |
| "logits/rejected": 0.48445090651512146, |
| "logps/chosen": -457.5303649902344, |
| "logps/rejected": -510.51959228515625, |
| "loss": 0.5521, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.668168067932129, |
| "rewards/margins": 0.9587725400924683, |
| "rewards/rejected": -2.6269407272338867, |
| "step": 2490 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.254796962702382e-06, |
| "logits/chosen": -0.9013687968254089, |
| "logits/rejected": -0.2077399045228958, |
| "logps/chosen": -482.5912170410156, |
| "logps/rejected": -530.4383544921875, |
| "loss": 0.5246, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9096496105194092, |
| "rewards/margins": 0.7448057532310486, |
| "rewards/rejected": -2.6544556617736816, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.33, |
| "eval_logits/chosen": 1.0913811922073364, |
| "eval_logits/rejected": 1.7388339042663574, |
| "eval_logps/chosen": -481.2728576660156, |
| "eval_logps/rejected": -538.2222290039062, |
| "eval_loss": 0.5413315296173096, |
| "eval_rewards/accuracies": 0.722000002861023, |
| "eval_rewards/chosen": -2.1278281211853027, |
| "eval_rewards/margins": 0.7685829997062683, |
| "eval_rewards/rejected": -2.896411180496216, |
| "eval_runtime": 2343.5019, |
| "eval_samples_per_second": 0.853, |
| "eval_steps_per_second": 0.427, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.246643053823864e-06, |
| "logits/chosen": -0.9218589663505554, |
| "logits/rejected": -0.20635871589183807, |
| "logps/chosen": -394.0392761230469, |
| "logps/rejected": -518.8317260742188, |
| "loss": 0.4924, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.8538624048233032, |
| "rewards/margins": 0.9797351956367493, |
| "rewards/rejected": -2.8335976600646973, |
| "step": 2510 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.238452683706979e-06, |
| "logits/chosen": -0.845422089099884, |
| "logits/rejected": -0.2671593129634857, |
| "logps/chosen": -385.7846984863281, |
| "logps/rejected": -433.94219970703125, |
| "loss": 0.52, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.65887451171875, |
| "rewards/margins": 0.8339220285415649, |
| "rewards/rejected": -2.4927964210510254, |
| "step": 2520 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.2302260233260025e-06, |
| "logits/chosen": -0.5951265096664429, |
| "logits/rejected": -0.30148234963417053, |
| "logps/chosen": -463.2898864746094, |
| "logps/rejected": -552.0485229492188, |
| "loss": 0.5021, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.882061243057251, |
| "rewards/margins": 0.9961883425712585, |
| "rewards/rejected": -2.8782496452331543, |
| "step": 2530 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.2219632444127766e-06, |
| "logits/chosen": -0.3708893358707428, |
| "logits/rejected": 0.35125669836997986, |
| "logps/chosen": -490.02197265625, |
| "logps/rejected": -541.1585083007812, |
| "loss": 0.577, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.2178242206573486, |
| "rewards/margins": 0.617692232131958, |
| "rewards/rejected": -2.8355164527893066, |
| "step": 2540 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.213664519453115e-06, |
| "logits/chosen": -0.8187354803085327, |
| "logits/rejected": -0.09626396000385284, |
| "logps/chosen": -438.7705993652344, |
| "logps/rejected": -519.1937866210938, |
| "loss": 0.5085, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.062185287475586, |
| "rewards/margins": 0.7500772476196289, |
| "rewards/rejected": -2.8122622966766357, |
| "step": 2550 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 4.205330021683208e-06, |
| "logits/chosen": -0.38596871495246887, |
| "logits/rejected": -0.07203111797571182, |
| "logps/chosen": -407.65740966796875, |
| "logps/rejected": -462.069580078125, |
| "loss": 0.5697, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.9310047626495361, |
| "rewards/margins": 0.5898799300193787, |
| "rewards/rejected": -2.5208847522735596, |
| "step": 2560 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.196959925086008e-06, |
| "logits/chosen": -0.22275564074516296, |
| "logits/rejected": -0.3874654173851013, |
| "logps/chosen": -478.38409423828125, |
| "logps/rejected": -548.2817993164062, |
| "loss": 0.6185, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.1326379776000977, |
| "rewards/margins": 0.47789937257766724, |
| "rewards/rejected": -2.6105377674102783, |
| "step": 2570 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.188554404387588e-06, |
| "logits/chosen": -1.0716884136199951, |
| "logits/rejected": -0.1740306168794632, |
| "logps/chosen": -451.58734130859375, |
| "logps/rejected": -492.9994201660156, |
| "loss": 0.5536, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7293052673339844, |
| "rewards/margins": 0.6498473286628723, |
| "rewards/rejected": -2.379152536392212, |
| "step": 2580 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.180113635053504e-06, |
| "logits/chosen": -0.3191063404083252, |
| "logits/rejected": -0.5359519720077515, |
| "logps/chosen": -417.9990234375, |
| "logps/rejected": -500.7484436035156, |
| "loss": 0.5768, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.6265789270401, |
| "rewards/margins": 0.6983104944229126, |
| "rewards/rejected": -2.3248894214630127, |
| "step": 2590 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.17163779328513e-06, |
| "logits/chosen": -0.8110774755477905, |
| "logits/rejected": -0.09047582000494003, |
| "logps/chosen": -430.71197509765625, |
| "logps/rejected": -488.92913818359375, |
| "loss": 0.5657, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.597516417503357, |
| "rewards/margins": 0.8392965197563171, |
| "rewards/rejected": -2.4368128776550293, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.34, |
| "eval_logits/chosen": 0.9886474609375, |
| "eval_logits/rejected": 1.657106876373291, |
| "eval_logps/chosen": -437.1171875, |
| "eval_logps/rejected": -495.00030517578125, |
| "eval_loss": 0.5373482704162598, |
| "eval_rewards/accuracies": 0.7200000286102295, |
| "eval_rewards/chosen": -1.6862711906433105, |
| "eval_rewards/margins": 0.7779201865196228, |
| "eval_rewards/rejected": -2.4641919136047363, |
| "eval_runtime": 2326.8602, |
| "eval_samples_per_second": 0.86, |
| "eval_steps_per_second": 0.43, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.163127056015975e-06, |
| "logits/chosen": -0.8432804346084595, |
| "logits/rejected": 0.2679641544818878, |
| "logps/chosen": -450.518798828125, |
| "logps/rejected": -520.7162475585938, |
| "loss": 0.5617, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.684958815574646, |
| "rewards/margins": 0.8352130651473999, |
| "rewards/rejected": -2.520171642303467, |
| "step": 2610 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.154581600907994e-06, |
| "logits/chosen": -0.8970314264297485, |
| "logits/rejected": -0.025129878893494606, |
| "logps/chosen": -394.2820739746094, |
| "logps/rejected": -448.968505859375, |
| "loss": 0.4855, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.448757529258728, |
| "rewards/margins": 0.8607563972473145, |
| "rewards/rejected": -2.309514284133911, |
| "step": 2620 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 4.14600160634788e-06, |
| "logits/chosen": -0.5631152391433716, |
| "logits/rejected": 0.03483830764889717, |
| "logps/chosen": -381.072998046875, |
| "logps/rejected": -490.3233947753906, |
| "loss": 0.4763, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.4302902221679688, |
| "rewards/margins": 1.0292136669158936, |
| "rewards/rejected": -2.459503650665283, |
| "step": 2630 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.137387251443335e-06, |
| "logits/chosen": -1.161237120628357, |
| "logits/rejected": 0.12660464644432068, |
| "logps/chosen": -383.56878662109375, |
| "logps/rejected": -413.07550048828125, |
| "loss": 0.5769, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.3406836986541748, |
| "rewards/margins": 0.6809174418449402, |
| "rewards/rejected": -2.0216009616851807, |
| "step": 2640 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.128738716019338e-06, |
| "logits/chosen": -0.9206587672233582, |
| "logits/rejected": 0.036871038377285004, |
| "logps/chosen": -433.82080078125, |
| "logps/rejected": -477.78619384765625, |
| "loss": 0.538, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.3911445140838623, |
| "rewards/margins": 0.7216758728027344, |
| "rewards/rejected": -2.1128203868865967, |
| "step": 2650 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.120056180614386e-06, |
| "logits/chosen": -0.46158695220947266, |
| "logits/rejected": 0.09717199206352234, |
| "logps/chosen": -426.7666015625, |
| "logps/rejected": -509.9091796875, |
| "loss": 0.5761, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.841655969619751, |
| "rewards/margins": 0.7033026814460754, |
| "rewards/rejected": -2.5449583530426025, |
| "step": 2660 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.111339826476725e-06, |
| "logits/chosen": -0.36191526055336, |
| "logits/rejected": 0.032123737037181854, |
| "logps/chosen": -385.82342529296875, |
| "logps/rejected": -472.90203857421875, |
| "loss": 0.5479, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.5523761510849, |
| "rewards/margins": 0.8306114077568054, |
| "rewards/rejected": -2.3829877376556396, |
| "step": 2670 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.102589835560572e-06, |
| "logits/chosen": -0.8637989163398743, |
| "logits/rejected": 0.42278608679771423, |
| "logps/chosen": -448.33477783203125, |
| "logps/rejected": -472.2781677246094, |
| "loss": 0.5531, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -1.3624870777130127, |
| "rewards/margins": 0.7671451568603516, |
| "rewards/rejected": -2.1296322345733643, |
| "step": 2680 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.09380639052231e-06, |
| "logits/chosen": -0.7388796210289001, |
| "logits/rejected": -0.16724643111228943, |
| "logps/chosen": -423.61370849609375, |
| "logps/rejected": -544.0777587890625, |
| "loss": 0.4631, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.392254114151001, |
| "rewards/margins": 1.0100640058517456, |
| "rewards/rejected": -2.402318239212036, |
| "step": 2690 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.084989674716679e-06, |
| "logits/chosen": -0.7967337369918823, |
| "logits/rejected": -0.30105361342430115, |
| "logps/chosen": -473.30743408203125, |
| "logps/rejected": -546.0299072265625, |
| "loss": 0.5216, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.9343668222427368, |
| "rewards/margins": 0.7889801859855652, |
| "rewards/rejected": -2.7233471870422363, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.35, |
| "eval_logits/chosen": 1.1290334463119507, |
| "eval_logits/rejected": 1.7935971021652222, |
| "eval_logps/chosen": -467.43646240234375, |
| "eval_logps/rejected": -522.52783203125, |
| "eval_loss": 0.5356955528259277, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -1.9894644021987915, |
| "eval_rewards/margins": 0.7500025629997253, |
| "eval_rewards/rejected": -2.7394673824310303, |
| "eval_runtime": 2319.9728, |
| "eval_samples_per_second": 0.862, |
| "eval_steps_per_second": 0.431, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 4.076139872192949e-06, |
| "logits/chosen": -0.7494713068008423, |
| "logits/rejected": 0.4128738045692444, |
| "logps/chosen": -523.4064331054688, |
| "logps/rejected": -553.7559204101562, |
| "loss": 0.5189, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.168044090270996, |
| "rewards/margins": 0.7787975668907166, |
| "rewards/rejected": -2.9468414783477783, |
| "step": 2710 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.067257167691074e-06, |
| "logits/chosen": -0.3000775873661041, |
| "logits/rejected": -0.04052892327308655, |
| "logps/chosen": -495.1270446777344, |
| "logps/rejected": -570.2074584960938, |
| "loss": 0.5173, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9619982242584229, |
| "rewards/margins": 0.9272588491439819, |
| "rewards/rejected": -2.8892569541931152, |
| "step": 2720 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.05834174663784e-06, |
| "logits/chosen": -0.5102322697639465, |
| "logits/rejected": -0.13270506262779236, |
| "logps/chosen": -456.03424072265625, |
| "logps/rejected": -465.28240966796875, |
| "loss": 0.6489, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.8180183172225952, |
| "rewards/margins": 0.48946094512939453, |
| "rewards/rejected": -2.307478904724121, |
| "step": 2730 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.0493937951429895e-06, |
| "logits/chosen": -1.0210916996002197, |
| "logits/rejected": -0.37836208939552307, |
| "logps/chosen": -429.5328674316406, |
| "logps/rejected": -458.05877685546875, |
| "loss": 0.5128, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.6570367813110352, |
| "rewards/margins": 0.6776058673858643, |
| "rewards/rejected": -2.3346428871154785, |
| "step": 2740 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.040413499995343e-06, |
| "logits/chosen": -0.7698782086372375, |
| "logits/rejected": -0.08163869380950928, |
| "logps/chosen": -478.63848876953125, |
| "logps/rejected": -546.338623046875, |
| "loss": 0.513, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.83940851688385, |
| "rewards/margins": 0.8054366111755371, |
| "rewards/rejected": -2.6448452472686768, |
| "step": 2750 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.031401048658892e-06, |
| "logits/chosen": -0.7496566772460938, |
| "logits/rejected": 0.024263203144073486, |
| "logps/chosen": -451.26336669921875, |
| "logps/rejected": -517.8736572265625, |
| "loss": 0.5583, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.7768245935440063, |
| "rewards/margins": 0.8688470721244812, |
| "rewards/rejected": -2.6456716060638428, |
| "step": 2760 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.022356629268894e-06, |
| "logits/chosen": -0.7827082276344299, |
| "logits/rejected": 0.21052603423595428, |
| "logps/chosen": -487.554443359375, |
| "logps/rejected": -506.66192626953125, |
| "loss": 0.6025, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.0982871055603027, |
| "rewards/margins": 0.5698203444480896, |
| "rewards/rejected": -2.668107509613037, |
| "step": 2770 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 4.013280430627936e-06, |
| "logits/chosen": -0.36198630928993225, |
| "logits/rejected": 0.06339599192142487, |
| "logps/chosen": -443.41961669921875, |
| "logps/rejected": -474.7898864746094, |
| "loss": 0.6083, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.9905750751495361, |
| "rewards/margins": 0.5316168665885925, |
| "rewards/rejected": -2.5221920013427734, |
| "step": 2780 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 4.004172642202002e-06, |
| "logits/chosen": -0.9610457420349121, |
| "logits/rejected": 0.30055952072143555, |
| "logps/chosen": -452.96343994140625, |
| "logps/rejected": -512.3440551757812, |
| "loss": 0.4909, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.0993430614471436, |
| "rewards/margins": 0.8472961187362671, |
| "rewards/rejected": -2.9466395378112793, |
| "step": 2790 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.995033454116512e-06, |
| "logits/chosen": -1.0286301374435425, |
| "logits/rejected": 0.016544628888368607, |
| "logps/chosen": -508.1040954589844, |
| "logps/rejected": -527.2821044921875, |
| "loss": 0.5865, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.1054210662841797, |
| "rewards/margins": 0.5004376173019409, |
| "rewards/rejected": -2.605858564376831, |
| "step": 2800 |
| }, |
| { |
| "epoch": 0.37, |
| "eval_logits/chosen": 1.101927399635315, |
| "eval_logits/rejected": 1.7565044164657593, |
| "eval_logps/chosen": -478.56048583984375, |
| "eval_logps/rejected": -529.6148681640625, |
| "eval_loss": 0.5350669622421265, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -2.1007049083709717, |
| "eval_rewards/margins": 0.709632158279419, |
| "eval_rewards/rejected": -2.8103370666503906, |
| "eval_runtime": 2335.1815, |
| "eval_samples_per_second": 0.856, |
| "eval_steps_per_second": 0.428, |
| "step": 2800 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.985863057152355e-06, |
| "logits/chosen": -0.4937034547328949, |
| "logits/rejected": -0.2508159577846527, |
| "logps/chosen": -504.6494140625, |
| "logps/rejected": -560.7249755859375, |
| "loss": 0.5349, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.059462785720825, |
| "rewards/margins": 0.8392888307571411, |
| "rewards/rejected": -2.898751735687256, |
| "step": 2810 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.976661642741908e-06, |
| "logits/chosen": -0.27275025844573975, |
| "logits/rejected": -0.03731584548950195, |
| "logps/chosen": -484.62506103515625, |
| "logps/rejected": -580.8031005859375, |
| "loss": 0.466, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.229395627975464, |
| "rewards/margins": 0.9002262353897095, |
| "rewards/rejected": -3.129621982574463, |
| "step": 2820 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.967429402965035e-06, |
| "logits/chosen": -0.3462119400501251, |
| "logits/rejected": 0.061771608889102936, |
| "logps/chosen": -553.1046142578125, |
| "logps/rejected": -620.2770385742188, |
| "loss": 0.5486, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.695598602294922, |
| "rewards/margins": 0.7296158671379089, |
| "rewards/rejected": -3.4252142906188965, |
| "step": 2830 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.958166530545085e-06, |
| "logits/chosen": -0.5360496044158936, |
| "logits/rejected": -0.30475491285324097, |
| "logps/chosen": -516.7517700195312, |
| "logps/rejected": -609.4689331054688, |
| "loss": 0.4753, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.5592031478881836, |
| "rewards/margins": 0.9313445091247559, |
| "rewards/rejected": -3.4905476570129395, |
| "step": 2840 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.948873218844863e-06, |
| "logits/chosen": 0.14160022139549255, |
| "logits/rejected": -0.38983091711997986, |
| "logps/chosen": -468.97314453125, |
| "logps/rejected": -537.5845947265625, |
| "loss": 0.6692, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -2.569286584854126, |
| "rewards/margins": 0.41470590233802795, |
| "rewards/rejected": -2.983992338180542, |
| "step": 2850 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.939549661862592e-06, |
| "logits/chosen": -0.5928257703781128, |
| "logits/rejected": 0.027815770357847214, |
| "logps/chosen": -490.8857421875, |
| "logps/rejected": -555.0617065429688, |
| "loss": 0.5306, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.194727659225464, |
| "rewards/margins": 0.8603199124336243, |
| "rewards/rejected": -3.0550477504730225, |
| "step": 2860 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.930196054227871e-06, |
| "logits/chosen": -0.7506051063537598, |
| "logits/rejected": 0.29753103852272034, |
| "logps/chosen": -445.02667236328125, |
| "logps/rejected": -515.992431640625, |
| "loss": 0.5096, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.015972375869751, |
| "rewards/margins": 0.8588827848434448, |
| "rewards/rejected": -2.8748552799224854, |
| "step": 2870 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.920812591197604e-06, |
| "logits/chosen": -1.051764726638794, |
| "logits/rejected": 0.04196419566869736, |
| "logps/chosen": -435.1283264160156, |
| "logps/rejected": -485.16290283203125, |
| "loss": 0.4647, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.730743646621704, |
| "rewards/margins": 0.8607555627822876, |
| "rewards/rejected": -2.591498851776123, |
| "step": 2880 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.9113994686519305e-06, |
| "logits/chosen": -1.111640214920044, |
| "logits/rejected": 0.03947020322084427, |
| "logps/chosen": -432.510009765625, |
| "logps/rejected": -520.13134765625, |
| "loss": 0.4645, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.6205533742904663, |
| "rewards/margins": 0.9038299322128296, |
| "rewards/rejected": -2.524383306503296, |
| "step": 2890 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.90195688309013e-06, |
| "logits/chosen": -0.9940659403800964, |
| "logits/rejected": -0.16780364513397217, |
| "logps/chosen": -408.54425048828125, |
| "logps/rejected": -470.9107360839844, |
| "loss": 0.5252, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.5781054496765137, |
| "rewards/margins": 0.9151102304458618, |
| "rewards/rejected": -2.493215560913086, |
| "step": 2900 |
| }, |
| { |
| "epoch": 0.38, |
| "eval_logits/chosen": 0.9107775688171387, |
| "eval_logits/rejected": 1.5685968399047852, |
| "eval_logps/chosen": -426.64959716796875, |
| "eval_logps/rejected": -492.73974609375, |
| "eval_loss": 0.537619948387146, |
| "eval_rewards/accuracies": 0.7204999923706055, |
| "eval_rewards/chosen": -1.581595540046692, |
| "eval_rewards/margins": 0.8599900603294373, |
| "eval_rewards/rejected": -2.4415855407714844, |
| "eval_runtime": 2334.4828, |
| "eval_samples_per_second": 0.857, |
| "eval_steps_per_second": 0.428, |
| "step": 2900 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.892485031626527e-06, |
| "logits/chosen": -1.0271751880645752, |
| "logits/rejected": -0.31663069128990173, |
| "logps/chosen": -413.0658264160156, |
| "logps/rejected": -492.5518493652344, |
| "loss": 0.5214, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.5365521907806396, |
| "rewards/margins": 0.9228584170341492, |
| "rewards/rejected": -2.4594106674194336, |
| "step": 2910 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.882984111986371e-06, |
| "logits/chosen": -0.7470510005950928, |
| "logits/rejected": 0.05860505253076553, |
| "logps/chosen": -435.0110778808594, |
| "logps/rejected": -458.22137451171875, |
| "loss": 0.5542, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.5745357275009155, |
| "rewards/margins": 0.6277254223823547, |
| "rewards/rejected": -2.202260971069336, |
| "step": 2920 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.873454322501711e-06, |
| "logits/chosen": -1.084609866142273, |
| "logits/rejected": -0.0922069326043129, |
| "logps/chosen": -392.8829650878906, |
| "logps/rejected": -459.1026306152344, |
| "loss": 0.5197, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.1698970794677734, |
| "rewards/margins": 0.9355286359786987, |
| "rewards/rejected": -2.1054255962371826, |
| "step": 2930 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.863895862107255e-06, |
| "logits/chosen": -1.0651941299438477, |
| "logits/rejected": -0.298669695854187, |
| "logps/chosen": -362.4504089355469, |
| "logps/rejected": -488.83001708984375, |
| "loss": 0.4332, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.0489132404327393, |
| "rewards/margins": 1.0638964176177979, |
| "rewards/rejected": -2.112809658050537, |
| "step": 2940 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.854308930336216e-06, |
| "logits/chosen": -1.077282190322876, |
| "logits/rejected": 0.2055933028459549, |
| "logps/chosen": -443.42486572265625, |
| "logps/rejected": -475.886474609375, |
| "loss": 0.555, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.3301843404769897, |
| "rewards/margins": 0.7858977317810059, |
| "rewards/rejected": -2.116081953048706, |
| "step": 2950 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.844693727316151e-06, |
| "logits/chosen": -1.1035977602005005, |
| "logits/rejected": -0.3125559389591217, |
| "logps/chosen": -408.77783203125, |
| "logps/rejected": -454.4917907714844, |
| "loss": 0.4932, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.2791414260864258, |
| "rewards/margins": 0.9063733220100403, |
| "rewards/rejected": -2.185514450073242, |
| "step": 2960 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.835050453764779e-06, |
| "logits/chosen": -0.4838363230228424, |
| "logits/rejected": -0.24740548431873322, |
| "logps/chosen": -358.820556640625, |
| "logps/rejected": -441.27001953125, |
| "loss": 0.4782, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.1172313690185547, |
| "rewards/margins": 1.0152322053909302, |
| "rewards/rejected": -2.1324634552001953, |
| "step": 2970 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.825379310985792e-06, |
| "logits/chosen": -0.7724876999855042, |
| "logits/rejected": -0.538825273513794, |
| "logps/chosen": -370.72052001953125, |
| "logps/rejected": -447.32745361328125, |
| "loss": 0.5206, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.1402572393417358, |
| "rewards/margins": 0.8123918771743774, |
| "rewards/rejected": -1.9526491165161133, |
| "step": 2980 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.815680500864651e-06, |
| "logits/chosen": -0.9769965410232544, |
| "logits/rejected": -0.2623330056667328, |
| "logps/chosen": -419.5028381347656, |
| "logps/rejected": -455.3551330566406, |
| "loss": 0.5107, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.1941817998886108, |
| "rewards/margins": 0.7795951962471008, |
| "rewards/rejected": -1.9737770557403564, |
| "step": 2990 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.80595422586438e-06, |
| "logits/chosen": -1.0406408309936523, |
| "logits/rejected": -0.20213142037391663, |
| "logps/chosen": -462.68487548828125, |
| "logps/rejected": -471.08123779296875, |
| "loss": 0.5381, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.3869965076446533, |
| "rewards/margins": 0.8180079460144043, |
| "rewards/rejected": -2.2050046920776367, |
| "step": 3000 |
| }, |
| { |
| "epoch": 0.39, |
| "eval_logits/chosen": 1.0232552289962769, |
| "eval_logits/rejected": 1.7206393480300903, |
| "eval_logps/chosen": -422.64849853515625, |
| "eval_logps/rejected": -485.7741394042969, |
| "eval_loss": 0.5305867791175842, |
| "eval_rewards/accuracies": 0.7229999899864197, |
| "eval_rewards/chosen": -1.5415852069854736, |
| "eval_rewards/margins": 0.8303446769714355, |
| "eval_rewards/rejected": -2.37192964553833, |
| "eval_runtime": 2335.0533, |
| "eval_samples_per_second": 0.857, |
| "eval_steps_per_second": 0.428, |
| "step": 3000 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.7962006890213266e-06, |
| "logits/chosen": -0.1788121610879898, |
| "logits/rejected": 0.016005922108888626, |
| "logps/chosen": -406.82928466796875, |
| "logps/rejected": -460.6402282714844, |
| "loss": 0.606, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.7624022960662842, |
| "rewards/margins": 0.5616916418075562, |
| "rewards/rejected": -2.324093818664551, |
| "step": 3010 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.7864200939409336e-06, |
| "logits/chosen": -1.0381033420562744, |
| "logits/rejected": 0.2535991072654724, |
| "logps/chosen": -414.68536376953125, |
| "logps/rejected": -450.0343322753906, |
| "loss": 0.5872, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.4555238485336304, |
| "rewards/margins": 0.6130931973457336, |
| "rewards/rejected": -2.068617105484009, |
| "step": 3020 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.7766126447934857e-06, |
| "logits/chosen": -1.1287479400634766, |
| "logits/rejected": -0.4950384199619293, |
| "logps/chosen": -377.2622985839844, |
| "logps/rejected": -427.8341369628906, |
| "loss": 0.5285, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.304473638534546, |
| "rewards/margins": 0.7007145881652832, |
| "rewards/rejected": -2.005188465118408, |
| "step": 3030 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.766778546309847e-06, |
| "logits/chosen": -0.7997163534164429, |
| "logits/rejected": 0.35035890340805054, |
| "logps/chosen": -444.89190673828125, |
| "logps/rejected": -414.5162048339844, |
| "loss": 0.6038, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.4366438388824463, |
| "rewards/margins": 0.5985267758369446, |
| "rewards/rejected": -2.035170793533325, |
| "step": 3040 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.7569180037771868e-06, |
| "logits/chosen": -0.36123308539390564, |
| "logits/rejected": -0.1611182987689972, |
| "logps/chosen": -421.418701171875, |
| "logps/rejected": -478.39300537109375, |
| "loss": 0.596, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.594327688217163, |
| "rewards/margins": 0.6372068524360657, |
| "rewards/rejected": -2.231534481048584, |
| "step": 3050 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.7470312230346955e-06, |
| "logits/chosen": -0.7029015421867371, |
| "logits/rejected": 0.3699846863746643, |
| "logps/chosen": -454.07025146484375, |
| "logps/rejected": -483.5018615722656, |
| "loss": 0.4709, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.4222877025604248, |
| "rewards/margins": 0.9277554750442505, |
| "rewards/rejected": -2.3500430583953857, |
| "step": 3060 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.7371184104692857e-06, |
| "logits/chosen": -1.1917140483856201, |
| "logits/rejected": -0.15570615231990814, |
| "logps/chosen": -475.3992614746094, |
| "logps/rejected": -481.27301025390625, |
| "loss": 0.5378, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.428065299987793, |
| "rewards/margins": 0.7733792066574097, |
| "rewards/rejected": -2.201444387435913, |
| "step": 3070 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.727179773011289e-06, |
| "logits/chosen": -0.5196498036384583, |
| "logits/rejected": -0.2388072907924652, |
| "logps/chosen": -431.4600524902344, |
| "logps/rejected": -482.206787109375, |
| "loss": 0.5579, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.5728769302368164, |
| "rewards/margins": 0.6688525080680847, |
| "rewards/rejected": -2.241729259490967, |
| "step": 3080 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.717215518130127e-06, |
| "logits/chosen": -0.6362151503562927, |
| "logits/rejected": 0.041982658207416534, |
| "logps/chosen": -410.38189697265625, |
| "logps/rejected": -457.52093505859375, |
| "loss": 0.5947, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.5848448276519775, |
| "rewards/margins": 0.5786422491073608, |
| "rewards/rejected": -2.163486957550049, |
| "step": 3090 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.7072258538299923e-06, |
| "logits/chosen": -1.2260688543319702, |
| "logits/rejected": 0.28254860639572144, |
| "logps/chosen": -483.26568603515625, |
| "logps/rejected": -465.8003845214844, |
| "loss": 0.4587, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.3665094375610352, |
| "rewards/margins": 0.8110405206680298, |
| "rewards/rejected": -2.1775500774383545, |
| "step": 3100 |
| }, |
| { |
| "epoch": 0.41, |
| "eval_logits/chosen": 1.1220719814300537, |
| "eval_logits/rejected": 1.844536542892456, |
| "eval_logps/chosen": -413.6004943847656, |
| "eval_logps/rejected": -467.0777587890625, |
| "eval_loss": 0.522217869758606, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -1.4511048793792725, |
| "eval_rewards/margins": 0.7338610291481018, |
| "eval_rewards/rejected": -2.1849660873413086, |
| "eval_runtime": 2334.682, |
| "eval_samples_per_second": 0.857, |
| "eval_steps_per_second": 0.428, |
| "step": 3100 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.6972109886454933e-06, |
| "logits/chosen": -0.25699383020401, |
| "logits/rejected": -0.20727062225341797, |
| "logps/chosen": -435.0664978027344, |
| "logps/rejected": -490.909912109375, |
| "loss": 0.4972, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.762265920639038, |
| "rewards/margins": 0.8372318148612976, |
| "rewards/rejected": -2.5994980335235596, |
| "step": 3110 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.687171131637314e-06, |
| "logits/chosen": -0.7965458035469055, |
| "logits/rejected": 0.2634788155555725, |
| "logps/chosen": -453.2422790527344, |
| "logps/rejected": -499.4004821777344, |
| "loss": 0.5227, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7745777368545532, |
| "rewards/margins": 0.7375408411026001, |
| "rewards/rejected": -2.5121185779571533, |
| "step": 3120 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.677106492387839e-06, |
| "logits/chosen": -0.7059360146522522, |
| "logits/rejected": 0.452791303396225, |
| "logps/chosen": -459.8872985839844, |
| "logps/rejected": -473.69781494140625, |
| "loss": 0.5509, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7924385070800781, |
| "rewards/margins": 0.7473533153533936, |
| "rewards/rejected": -2.5397915840148926, |
| "step": 3130 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.6670172809967865e-06, |
| "logits/chosen": -0.48607778549194336, |
| "logits/rejected": 0.2857428193092346, |
| "logps/chosen": -408.91790771484375, |
| "logps/rejected": -458.8837890625, |
| "loss": 0.5115, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.9582751989364624, |
| "rewards/margins": 0.6871889233589172, |
| "rewards/rejected": -2.6454639434814453, |
| "step": 3140 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.6569037080768153e-06, |
| "logits/chosen": -0.8563801646232605, |
| "logits/rejected": -0.1331445276737213, |
| "logps/chosen": -419.33544921875, |
| "logps/rejected": -523.2894287109375, |
| "loss": 0.5028, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7469348907470703, |
| "rewards/margins": 0.928310215473175, |
| "rewards/rejected": -2.6752450466156006, |
| "step": 3150 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.646765984749137e-06, |
| "logits/chosen": -0.42074188590049744, |
| "logits/rejected": -0.15325963497161865, |
| "logps/chosen": -437.05352783203125, |
| "logps/rejected": -532.8778076171875, |
| "loss": 0.4926, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.6952593326568604, |
| "rewards/margins": 0.9555587768554688, |
| "rewards/rejected": -2.650818109512329, |
| "step": 3160 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.6366043226391e-06, |
| "logits/chosen": -0.8013358116149902, |
| "logits/rejected": 0.14026977121829987, |
| "logps/chosen": -455.299560546875, |
| "logps/rejected": -499.14892578125, |
| "loss": 0.505, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.768510103225708, |
| "rewards/margins": 0.9221822023391724, |
| "rewards/rejected": -2.690692186355591, |
| "step": 3170 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.6264189338717766e-06, |
| "logits/chosen": -1.1270676851272583, |
| "logits/rejected": 0.0055741192772984505, |
| "logps/chosen": -444.71502685546875, |
| "logps/rejected": -478.0035705566406, |
| "loss": 0.5603, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.6736608743667603, |
| "rewards/margins": 0.6656028032302856, |
| "rewards/rejected": -2.339263439178467, |
| "step": 3180 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.6162100310675334e-06, |
| "logits/chosen": -0.5859667062759399, |
| "logits/rejected": -0.5138736963272095, |
| "logps/chosen": -400.99151611328125, |
| "logps/rejected": -445.34747314453125, |
| "loss": 0.65, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.3842413425445557, |
| "rewards/margins": 0.5574811697006226, |
| "rewards/rejected": -1.9417225122451782, |
| "step": 3190 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.605977827337596e-06, |
| "logits/chosen": -0.5511821508407593, |
| "logits/rejected": -0.18979701399803162, |
| "logps/chosen": -392.2962341308594, |
| "logps/rejected": -460.0108947753906, |
| "loss": 0.5173, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.33115553855896, |
| "rewards/margins": 0.8485208749771118, |
| "rewards/rejected": -2.1796765327453613, |
| "step": 3200 |
| }, |
| { |
| "epoch": 0.42, |
| "eval_logits/chosen": 0.8981449604034424, |
| "eval_logits/rejected": 1.6186491250991821, |
| "eval_logps/chosen": -403.9989318847656, |
| "eval_logps/rejected": -462.40948486328125, |
| "eval_loss": 0.5276510715484619, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -1.3550893068313599, |
| "eval_rewards/margins": 0.7831941843032837, |
| "eval_rewards/rejected": -2.1382837295532227, |
| "eval_runtime": 2082.617, |
| "eval_samples_per_second": 0.96, |
| "eval_steps_per_second": 0.48, |
| "step": 3200 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.595722536279595e-06, |
| "logits/chosen": -1.2897363901138306, |
| "logits/rejected": 0.7435730695724487, |
| "logps/chosen": -463.46563720703125, |
| "logps/rejected": -484.604736328125, |
| "loss": 0.4551, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.345106840133667, |
| "rewards/margins": 0.9554675221443176, |
| "rewards/rejected": -2.300574541091919, |
| "step": 3210 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.58544437197311e-06, |
| "logits/chosen": -0.5996850728988647, |
| "logits/rejected": 0.24509985744953156, |
| "logps/chosen": -428.47125244140625, |
| "logps/rejected": -500.66168212890625, |
| "loss": 0.5034, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.491923213005066, |
| "rewards/margins": 0.9786638021469116, |
| "rewards/rejected": -2.4705870151519775, |
| "step": 3220 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.5751435489752025e-06, |
| "logits/chosen": -0.4525715410709381, |
| "logits/rejected": 0.17086896300315857, |
| "logps/chosen": -396.47723388671875, |
| "logps/rejected": -455.05706787109375, |
| "loss": 0.4999, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.4165663719177246, |
| "rewards/margins": 0.9099749326705933, |
| "rewards/rejected": -2.3265414237976074, |
| "step": 3230 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.5648202823159317e-06, |
| "logits/chosen": -0.16280296444892883, |
| "logits/rejected": 0.15849009156227112, |
| "logps/chosen": -411.54742431640625, |
| "logps/rejected": -548.3161010742188, |
| "loss": 0.4644, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.7535407543182373, |
| "rewards/margins": 1.087467908859253, |
| "rewards/rejected": -2.8410089015960693, |
| "step": 3240 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.554474787493873e-06, |
| "logits/chosen": 0.05241694301366806, |
| "logits/rejected": 0.6363898515701294, |
| "logps/chosen": -502.8150329589844, |
| "logps/rejected": -597.0306396484375, |
| "loss": 0.4855, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.0513508319854736, |
| "rewards/margins": 1.1114223003387451, |
| "rewards/rejected": -3.1627731323242188, |
| "step": 3250 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.5441072804716125e-06, |
| "logits/chosen": 0.01650853082537651, |
| "logits/rejected": 0.25886648893356323, |
| "logps/chosen": -555.9203491210938, |
| "logps/rejected": -650.7965087890625, |
| "loss": 0.5809, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.538379192352295, |
| "rewards/margins": 0.8736263513565063, |
| "rewards/rejected": -3.412005662918091, |
| "step": 3260 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.5337179776712427e-06, |
| "logits/chosen": -0.008978593163192272, |
| "logits/rejected": 0.6548370718955994, |
| "logps/chosen": -530.6326904296875, |
| "logps/rejected": -644.48876953125, |
| "loss": 0.5841, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.7416481971740723, |
| "rewards/margins": 1.1851587295532227, |
| "rewards/rejected": -3.926807403564453, |
| "step": 3270 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.5233070959698445e-06, |
| "logits/chosen": -0.5847116708755493, |
| "logits/rejected": 0.3420669138431549, |
| "logps/chosen": -555.023681640625, |
| "logps/rejected": -576.3981323242188, |
| "loss": 0.6021, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.538581371307373, |
| "rewards/margins": 0.596244752407074, |
| "rewards/rejected": -3.1348259449005127, |
| "step": 3280 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.512874852694959e-06, |
| "logits/chosen": -0.6708934903144836, |
| "logits/rejected": 0.5892789959907532, |
| "logps/chosen": -480.2303161621094, |
| "logps/rejected": -554.13671875, |
| "loss": 0.4727, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.084829807281494, |
| "rewards/margins": 0.9657213091850281, |
| "rewards/rejected": -3.050551414489746, |
| "step": 3290 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.5024214656200497e-06, |
| "logits/chosen": -0.9602855443954468, |
| "logits/rejected": 0.5978536009788513, |
| "logps/chosen": -469.49737548828125, |
| "logps/rejected": -488.523681640625, |
| "loss": 0.5851, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.8592431545257568, |
| "rewards/margins": 0.7517732381820679, |
| "rewards/rejected": -2.6110165119171143, |
| "step": 3300 |
| }, |
| { |
| "epoch": 0.43, |
| "eval_logits/chosen": 1.2859539985656738, |
| "eval_logits/rejected": 2.034395933151245, |
| "eval_logps/chosen": -437.1257629394531, |
| "eval_logps/rejected": -498.6931457519531, |
| "eval_loss": 0.5180677771568298, |
| "eval_rewards/accuracies": 0.7325000166893005, |
| "eval_rewards/chosen": -1.6863574981689453, |
| "eval_rewards/margins": 0.8147625923156738, |
| "eval_rewards/rejected": -2.50111985206604, |
| "eval_runtime": 1144.5777, |
| "eval_samples_per_second": 1.747, |
| "eval_steps_per_second": 0.874, |
| "step": 3300 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.491947152959958e-06, |
| "logits/chosen": -0.5393772125244141, |
| "logits/rejected": 0.08978531509637833, |
| "logps/chosen": -471.2378845214844, |
| "logps/rejected": -527.0679321289062, |
| "loss": 0.5259, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7623119354248047, |
| "rewards/margins": 0.7789031863212585, |
| "rewards/rejected": -2.541214942932129, |
| "step": 3310 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.4814521333663497e-06, |
| "logits/chosen": -0.8797744512557983, |
| "logits/rejected": 0.28487175703048706, |
| "logps/chosen": -507.973388671875, |
| "logps/rejected": -497.87677001953125, |
| "loss": 0.5358, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7715896368026733, |
| "rewards/margins": 0.7174810767173767, |
| "rewards/rejected": -2.489070415496826, |
| "step": 3320 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4709366259231468e-06, |
| "logits/chosen": -0.5718088150024414, |
| "logits/rejected": 0.6245108842849731, |
| "logps/chosen": -451.671142578125, |
| "logps/rejected": -492.45111083984375, |
| "loss": 0.5618, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.632046103477478, |
| "rewards/margins": 0.802625834941864, |
| "rewards/rejected": -2.4346723556518555, |
| "step": 3330 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.460400850141956e-06, |
| "logits/chosen": -0.8436506390571594, |
| "logits/rejected": 0.6500002145767212, |
| "logps/chosen": -418.16571044921875, |
| "logps/rejected": -476.41375732421875, |
| "loss": 0.5184, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.8505115509033203, |
| "rewards/margins": 0.7895227670669556, |
| "rewards/rejected": -2.6400341987609863, |
| "step": 3340 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4498450259574858e-06, |
| "logits/chosen": -0.3697187304496765, |
| "logits/rejected": 0.12063203006982803, |
| "logps/chosen": -465.37066650390625, |
| "logps/rejected": -506.58319091796875, |
| "loss": 0.5763, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.957269310951233, |
| "rewards/margins": 0.574730634689331, |
| "rewards/rejected": -2.5320000648498535, |
| "step": 3350 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.439269373722957e-06, |
| "logits/chosen": -0.4114798605442047, |
| "logits/rejected": 0.28758490085601807, |
| "logps/chosen": -450.16741943359375, |
| "logps/rejected": -514.9403076171875, |
| "loss": 0.542, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.925722360610962, |
| "rewards/margins": 0.8905431628227234, |
| "rewards/rejected": -2.81626558303833, |
| "step": 3360 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4286741142055014e-06, |
| "logits/chosen": -0.9048460721969604, |
| "logits/rejected": -0.23542626202106476, |
| "logps/chosen": -464.1852111816406, |
| "logps/rejected": -529.8837890625, |
| "loss": 0.4867, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.771493673324585, |
| "rewards/margins": 0.8255535364151001, |
| "rewards/rejected": -2.5970473289489746, |
| "step": 3370 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4180594685815536e-06, |
| "logits/chosen": -0.6555899381637573, |
| "logits/rejected": 0.23162047564983368, |
| "logps/chosen": -387.56414794921875, |
| "logps/rejected": -469.15020751953125, |
| "loss": 0.5231, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.6515061855316162, |
| "rewards/margins": 0.8171674609184265, |
| "rewards/rejected": -2.4686737060546875, |
| "step": 3380 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4074256584322336e-06, |
| "logits/chosen": -0.6201430559158325, |
| "logits/rejected": 0.13968434929847717, |
| "logps/chosen": -395.79229736328125, |
| "logps/rejected": -470.4158630371094, |
| "loss": 0.521, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.4998222589492798, |
| "rewards/margins": 0.9599917531013489, |
| "rewards/rejected": -2.4598140716552734, |
| "step": 3390 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.3967729057387213e-06, |
| "logits/chosen": -0.7700475454330444, |
| "logits/rejected": 0.4305594563484192, |
| "logps/chosen": -439.98504638671875, |
| "logps/rejected": -465.68408203125, |
| "loss": 0.5811, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.470069169998169, |
| "rewards/margins": 0.6720464825630188, |
| "rewards/rejected": -2.142115831375122, |
| "step": 3400 |
| }, |
| { |
| "epoch": 0.44, |
| "eval_logits/chosen": 1.0161676406860352, |
| "eval_logits/rejected": 1.7237727642059326, |
| "eval_logps/chosen": -428.5589904785156, |
| "eval_logps/rejected": -492.4408264160156, |
| "eval_loss": 0.5165792107582092, |
| "eval_rewards/accuracies": 0.7335000038146973, |
| "eval_rewards/chosen": -1.6006896495819092, |
| "eval_rewards/margins": 0.8379069566726685, |
| "eval_rewards/rejected": -2.4385969638824463, |
| "eval_runtime": 1168.883, |
| "eval_samples_per_second": 1.711, |
| "eval_steps_per_second": 0.856, |
| "step": 3400 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.386101432877624e-06, |
| "logits/chosen": -0.7892847657203674, |
| "logits/rejected": -0.024471605196595192, |
| "logps/chosen": -428.40667724609375, |
| "logps/rejected": -461.5455627441406, |
| "loss": 0.5302, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.6095516681671143, |
| "rewards/margins": 0.7331417798995972, |
| "rewards/rejected": -2.342693328857422, |
| "step": 3410 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.375411462616332e-06, |
| "logits/chosen": -1.0004537105560303, |
| "logits/rejected": 0.13648569583892822, |
| "logps/chosen": -466.4956970214844, |
| "logps/rejected": -552.7140502929688, |
| "loss": 0.5037, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.7703803777694702, |
| "rewards/margins": 0.8303622007369995, |
| "rewards/rejected": -2.600742816925049, |
| "step": 3420 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.3647032181083696e-06, |
| "logits/chosen": -0.7642697095870972, |
| "logits/rejected": -0.002010262105613947, |
| "logps/chosen": -486.2002868652344, |
| "logps/rejected": -559.04736328125, |
| "loss": 0.4941, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.827528715133667, |
| "rewards/margins": 0.9090808629989624, |
| "rewards/rejected": -2.73660945892334, |
| "step": 3430 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.3539769228887382e-06, |
| "logits/chosen": -0.9899004697799683, |
| "logits/rejected": 0.285744845867157, |
| "logps/chosen": -455.26788330078125, |
| "logps/rejected": -538.8175048828125, |
| "loss": 0.4733, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.473595142364502, |
| "rewards/margins": 0.9281530380249023, |
| "rewards/rejected": -2.4017484188079834, |
| "step": 3440 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.343232800869247e-06, |
| "logits/chosen": -0.8719793558120728, |
| "logits/rejected": 0.3464857041835785, |
| "logps/chosen": -392.4674072265625, |
| "logps/rejected": -419.7901306152344, |
| "loss": 0.518, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.578559160232544, |
| "rewards/margins": 0.7874252796173096, |
| "rewards/rejected": -2.3659844398498535, |
| "step": 3450 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.33247107633384e-06, |
| "logits/chosen": -0.6414632201194763, |
| "logits/rejected": -0.19829490780830383, |
| "logps/chosen": -397.5970764160156, |
| "logps/rejected": -498.804931640625, |
| "loss": 0.4297, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -1.311751365661621, |
| "rewards/margins": 1.1057935953140259, |
| "rewards/rejected": -2.4175448417663574, |
| "step": 3460 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.3216919739339155e-06, |
| "logits/chosen": -0.8113977313041687, |
| "logits/rejected": 0.24353833496570587, |
| "logps/chosen": -451.53900146484375, |
| "logps/rejected": -521.3798217773438, |
| "loss": 0.4028, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.5756160020828247, |
| "rewards/margins": 1.230313777923584, |
| "rewards/rejected": -2.805929660797119, |
| "step": 3470 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.310895718683635e-06, |
| "logits/chosen": -0.5552123188972473, |
| "logits/rejected": 0.25212961435317993, |
| "logps/chosen": -468.75030517578125, |
| "logps/rejected": -510.51214599609375, |
| "loss": 0.6551, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.7728900909423828, |
| "rewards/margins": 0.6345969438552856, |
| "rewards/rejected": -2.4074866771698, |
| "step": 3480 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.3000825359552256e-06, |
| "logits/chosen": -0.29439646005630493, |
| "logits/rejected": 0.14724037051200867, |
| "logps/chosen": -405.78594970703125, |
| "logps/rejected": -501.95465087890625, |
| "loss": 0.497, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.3883360624313354, |
| "rewards/margins": 0.9358784556388855, |
| "rewards/rejected": -2.324214458465576, |
| "step": 3490 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.2892526514742778e-06, |
| "logits/chosen": -0.5424224734306335, |
| "logits/rejected": 0.3168255388736725, |
| "logps/chosen": -418.60272216796875, |
| "logps/rejected": -471.0479431152344, |
| "loss": 0.4892, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.4777576923370361, |
| "rewards/margins": 0.914044201374054, |
| "rewards/rejected": -2.391801595687866, |
| "step": 3500 |
| }, |
| { |
| "epoch": 0.46, |
| "eval_logits/chosen": 1.3014029264450073, |
| "eval_logits/rejected": 2.0708816051483154, |
| "eval_logps/chosen": -415.6103820800781, |
| "eval_logps/rejected": -480.9519348144531, |
| "eval_loss": 0.5256860256195068, |
| "eval_rewards/accuracies": 0.7279999852180481, |
| "eval_rewards/chosen": -1.471203327178955, |
| "eval_rewards/margins": 0.8525046110153198, |
| "eval_rewards/rejected": -2.3237080574035645, |
| "eval_runtime": 1172.9577, |
| "eval_samples_per_second": 1.705, |
| "eval_steps_per_second": 0.853, |
| "step": 3500 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.27840629131503e-06, |
| "logits/chosen": -0.525278627872467, |
| "logits/rejected": 0.5383912324905396, |
| "logps/chosen": -441.439208984375, |
| "logps/rejected": -500.04010009765625, |
| "loss": 0.5312, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.6438672542572021, |
| "rewards/margins": 0.845868706703186, |
| "rewards/rejected": -2.4897360801696777, |
| "step": 3510 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.2675436818956522e-06, |
| "logits/chosen": -0.6424092054367065, |
| "logits/rejected": 0.2780497372150421, |
| "logps/chosen": -387.6494445800781, |
| "logps/rejected": -470.696533203125, |
| "loss": 0.5343, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.4292685985565186, |
| "rewards/margins": 0.742842972278595, |
| "rewards/rejected": -2.1721115112304688, |
| "step": 3520 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.2566650499735185e-06, |
| "logits/chosen": -0.35371822118759155, |
| "logits/rejected": 0.6347888112068176, |
| "logps/chosen": -437.46600341796875, |
| "logps/rejected": -535.9605712890625, |
| "loss": 0.4477, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.4554798603057861, |
| "rewards/margins": 1.2568625211715698, |
| "rewards/rejected": -2.7123425006866455, |
| "step": 3530 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.2457706226404715e-06, |
| "logits/chosen": -0.41827550530433655, |
| "logits/rejected": -0.005315917544066906, |
| "logps/chosen": -431.5044860839844, |
| "logps/rejected": -449.8809509277344, |
| "loss": 0.6296, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -1.6185357570648193, |
| "rewards/margins": 0.5960680246353149, |
| "rewards/rejected": -2.214603900909424, |
| "step": 3540 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.2348606273180847e-06, |
| "logits/chosen": -1.0620290040969849, |
| "logits/rejected": 0.8680380582809448, |
| "logps/chosen": -446.9424743652344, |
| "logps/rejected": -452.28485107421875, |
| "loss": 0.4631, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.3773703575134277, |
| "rewards/margins": 0.9029923677444458, |
| "rewards/rejected": -2.280362844467163, |
| "step": 3550 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.2239352917529165e-06, |
| "logits/chosen": -0.7291229367256165, |
| "logits/rejected": 0.35278087854385376, |
| "logps/chosen": -489.33551025390625, |
| "logps/rejected": -550.3321533203125, |
| "loss": 0.519, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8307793140411377, |
| "rewards/margins": 0.7998561859130859, |
| "rewards/rejected": -2.6306357383728027, |
| "step": 3560 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.2129948440117487e-06, |
| "logits/chosen": -0.4160127639770508, |
| "logits/rejected": 0.06812303513288498, |
| "logps/chosen": -480.4317932128906, |
| "logps/rejected": -528.1541748046875, |
| "loss": 0.5639, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.1663801670074463, |
| "rewards/margins": 0.6297258734703064, |
| "rewards/rejected": -2.7961058616638184, |
| "step": 3570 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.202039512476833e-06, |
| "logits/chosen": -0.47289180755615234, |
| "logits/rejected": 0.3967745304107666, |
| "logps/chosen": -413.58563232421875, |
| "logps/rejected": -521.4444580078125, |
| "loss": 0.4684, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.8022072315216064, |
| "rewards/margins": 1.0440325736999512, |
| "rewards/rejected": -2.8462398052215576, |
| "step": 3580 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.1910695258411216e-06, |
| "logits/chosen": -0.7860490679740906, |
| "logits/rejected": 0.8082360029220581, |
| "logps/chosen": -440.5712890625, |
| "logps/rejected": -458.45526123046875, |
| "loss": 0.5925, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7098718881607056, |
| "rewards/margins": 0.7629823684692383, |
| "rewards/rejected": -2.4728541374206543, |
| "step": 3590 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.1800851131034904e-06, |
| "logits/chosen": -0.5506579279899597, |
| "logits/rejected": 0.5083298087120056, |
| "logps/chosen": -437.36016845703125, |
| "logps/rejected": -497.79742431640625, |
| "loss": 0.5438, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7282190322875977, |
| "rewards/margins": 0.9188164472579956, |
| "rewards/rejected": -2.647035598754883, |
| "step": 3600 |
| }, |
| { |
| "epoch": 0.47, |
| "eval_logits/chosen": 1.4150185585021973, |
| "eval_logits/rejected": 2.2019591331481934, |
| "eval_logps/chosen": -428.1592102050781, |
| "eval_logps/rejected": -493.0663757324219, |
| "eval_loss": 0.5251966714859009, |
| "eval_rewards/accuracies": 0.7275000214576721, |
| "eval_rewards/chosen": -1.5966919660568237, |
| "eval_rewards/margins": 0.8481603860855103, |
| "eval_rewards/rejected": -2.444852113723755, |
| "eval_runtime": 1146.1005, |
| "eval_samples_per_second": 1.745, |
| "eval_steps_per_second": 0.873, |
| "step": 3600 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.169086503563962e-06, |
| "logits/chosen": -0.6776447296142578, |
| "logits/rejected": -0.23693008720874786, |
| "logps/chosen": -416.4723205566406, |
| "logps/rejected": -498.85479736328125, |
| "loss": 0.6094, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -1.5892466306686401, |
| "rewards/margins": 0.6511684656143188, |
| "rewards/rejected": -2.240415096282959, |
| "step": 3610 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.1580739268189165e-06, |
| "logits/chosen": -0.5585545897483826, |
| "logits/rejected": 0.5914020538330078, |
| "logps/chosen": -437.05572509765625, |
| "logps/rejected": -491.81671142578125, |
| "loss": 0.505, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.594426155090332, |
| "rewards/margins": 0.934490978717804, |
| "rewards/rejected": -2.528916835784912, |
| "step": 3620 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.147047612756302e-06, |
| "logits/chosen": -0.24671559035778046, |
| "logits/rejected": 0.12607893347740173, |
| "logps/chosen": -449.9618225097656, |
| "logps/rejected": -528.7197265625, |
| "loss": 0.4698, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.4847663640975952, |
| "rewards/margins": 0.9548713564872742, |
| "rewards/rejected": -2.4396376609802246, |
| "step": 3630 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.136007791550833e-06, |
| "logits/chosen": -0.7207472324371338, |
| "logits/rejected": 0.6024399995803833, |
| "logps/chosen": -402.71319580078125, |
| "logps/rejected": -450.76373291015625, |
| "loss": 0.5079, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.6094977855682373, |
| "rewards/margins": 0.8133002519607544, |
| "rewards/rejected": -2.4227981567382812, |
| "step": 3640 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.1249546936591848e-06, |
| "logits/chosen": -0.587475597858429, |
| "logits/rejected": 0.040339358150959015, |
| "logps/chosen": -386.5211181640625, |
| "logps/rejected": -455.6336364746094, |
| "loss": 0.5345, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.4895503520965576, |
| "rewards/margins": 0.7380887866020203, |
| "rewards/rejected": -2.2276394367218018, |
| "step": 3650 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.1138885498151843e-06, |
| "logits/chosen": -0.3841520845890045, |
| "logits/rejected": 0.43626269698143005, |
| "logps/chosen": -448.64080810546875, |
| "logps/rejected": -513.5535888671875, |
| "loss": 0.4928, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.7264331579208374, |
| "rewards/margins": 1.076446771621704, |
| "rewards/rejected": -2.802879810333252, |
| "step": 3660 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.1028095910249937e-06, |
| "logits/chosen": -1.0335357189178467, |
| "logits/rejected": 0.6055603623390198, |
| "logps/chosen": -452.31829833984375, |
| "logps/rejected": -487.44921875, |
| "loss": 0.4924, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7537025213241577, |
| "rewards/margins": 0.8785789608955383, |
| "rewards/rejected": -2.632281541824341, |
| "step": 3670 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.0917180485622895e-06, |
| "logits/chosen": -0.4549393653869629, |
| "logits/rejected": 1.1361083984375, |
| "logps/chosen": -451.30145263671875, |
| "logps/rejected": -507.1758728027344, |
| "loss": 0.4922, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.7780840396881104, |
| "rewards/margins": 1.0487792491912842, |
| "rewards/rejected": -2.8268632888793945, |
| "step": 3680 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.0806141539634294e-06, |
| "logits/chosen": -0.6923087239265442, |
| "logits/rejected": 0.8231356739997864, |
| "logps/chosen": -423.26531982421875, |
| "logps/rejected": -463.19921875, |
| "loss": 0.5082, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7002700567245483, |
| "rewards/margins": 0.9019443392753601, |
| "rewards/rejected": -2.6022145748138428, |
| "step": 3690 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.069498139022624e-06, |
| "logits/chosen": -0.8322190046310425, |
| "logits/rejected": 0.6718935966491699, |
| "logps/chosen": -491.6170349121094, |
| "logps/rejected": -499.7591247558594, |
| "loss": 0.5677, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.0136921405792236, |
| "rewards/margins": 0.6361916661262512, |
| "rewards/rejected": -2.64988374710083, |
| "step": 3700 |
| }, |
| { |
| "epoch": 0.48, |
| "eval_logits/chosen": 1.6843464374542236, |
| "eval_logits/rejected": 2.467794895172119, |
| "eval_logps/chosen": -465.7503662109375, |
| "eval_logps/rejected": -529.8629760742188, |
| "eval_loss": 0.5152121782302856, |
| "eval_rewards/accuracies": 0.7275000214576721, |
| "eval_rewards/chosen": -1.972603678703308, |
| "eval_rewards/margins": 0.8402146697044373, |
| "eval_rewards/rejected": -2.8128180503845215, |
| "eval_runtime": 1146.5714, |
| "eval_samples_per_second": 1.744, |
| "eval_steps_per_second": 0.872, |
| "step": 3700 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 3.0583702357870964e-06, |
| "logits/chosen": -0.48846787214279175, |
| "logits/rejected": -0.0028875707648694515, |
| "logps/chosen": -522.038818359375, |
| "logps/rejected": -585.753662109375, |
| "loss": 0.6047, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.117889881134033, |
| "rewards/margins": 0.6571815609931946, |
| "rewards/rejected": -2.775071144104004, |
| "step": 3710 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 3.0472306765522393e-06, |
| "logits/chosen": -0.9246646761894226, |
| "logits/rejected": 1.0386006832122803, |
| "logps/chosen": -434.03704833984375, |
| "logps/rejected": -522.7357177734375, |
| "loss": 0.4683, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8714182376861572, |
| "rewards/margins": 1.1093872785568237, |
| "rewards/rejected": -2.9808051586151123, |
| "step": 3720 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 3.0360796938567628e-06, |
| "logits/chosen": -0.7488586902618408, |
| "logits/rejected": 0.5929322838783264, |
| "logps/chosen": -465.7920837402344, |
| "logps/rejected": -497.0887145996094, |
| "loss": 0.5484, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9402759075164795, |
| "rewards/margins": 0.7168923616409302, |
| "rewards/rejected": -2.65716814994812, |
| "step": 3730 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 3.0249175204778435e-06, |
| "logits/chosen": -0.12813475728034973, |
| "logits/rejected": 0.1722133457660675, |
| "logps/chosen": -443.7284240722656, |
| "logps/rejected": -510.58660888671875, |
| "loss": 0.513, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8336127996444702, |
| "rewards/margins": 0.8592530488967896, |
| "rewards/rejected": -2.6928658485412598, |
| "step": 3740 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 3.0137443894262634e-06, |
| "logits/chosen": -0.1264442503452301, |
| "logits/rejected": 1.0170029401779175, |
| "logps/chosen": -455.2422790527344, |
| "logps/rejected": -530.8844604492188, |
| "loss": 0.3586, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -1.7252286672592163, |
| "rewards/margins": 1.3205913305282593, |
| "rewards/rejected": -3.0458197593688965, |
| "step": 3750 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 3.0025605339415476e-06, |
| "logits/chosen": -0.2637806832790375, |
| "logits/rejected": 0.7203923463821411, |
| "logps/chosen": -443.43988037109375, |
| "logps/rejected": -518.1168823242188, |
| "loss": 0.4633, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.7762037515640259, |
| "rewards/margins": 1.030763030052185, |
| "rewards/rejected": -2.8069663047790527, |
| "step": 3760 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 2.9913661874870923e-06, |
| "logits/chosen": -0.07495728135108948, |
| "logits/rejected": 0.4468969702720642, |
| "logps/chosen": -466.39678955078125, |
| "logps/rejected": -520.4652709960938, |
| "loss": 0.5005, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.0380637645721436, |
| "rewards/margins": 0.8805074691772461, |
| "rewards/rejected": -2.9185712337493896, |
| "step": 3770 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 2.980161583745294e-06, |
| "logits/chosen": -0.31729915738105774, |
| "logits/rejected": 0.5375093221664429, |
| "logps/chosen": -503.6490783691406, |
| "logps/rejected": -559.5162963867188, |
| "loss": 0.4487, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9461323022842407, |
| "rewards/margins": 1.0691391229629517, |
| "rewards/rejected": -3.0152714252471924, |
| "step": 3780 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.96894695661267e-06, |
| "logits/chosen": -0.8055630922317505, |
| "logits/rejected": 0.4190472662448883, |
| "logps/chosen": -503.9349670410156, |
| "logps/rejected": -536.4412841796875, |
| "loss": 0.5495, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.072908878326416, |
| "rewards/margins": 0.787026584148407, |
| "rewards/rejected": -2.8599355220794678, |
| "step": 3790 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.9577225401949773e-06, |
| "logits/chosen": -0.2834416925907135, |
| "logits/rejected": -0.048535846173763275, |
| "logps/chosen": -445.82147216796875, |
| "logps/rejected": -526.7498779296875, |
| "loss": 0.5471, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.1524107456207275, |
| "rewards/margins": 0.9361416101455688, |
| "rewards/rejected": -3.0885519981384277, |
| "step": 3800 |
| }, |
| { |
| "epoch": 0.5, |
| "eval_logits/chosen": 1.4351999759674072, |
| "eval_logits/rejected": 2.202155113220215, |
| "eval_logps/chosen": -475.79779052734375, |
| "eval_logps/rejected": -551.583251953125, |
| "eval_loss": 0.5240182876586914, |
| "eval_rewards/accuracies": 0.7254999876022339, |
| "eval_rewards/chosen": -2.07307767868042, |
| "eval_rewards/margins": 0.956943154335022, |
| "eval_rewards/rejected": -3.0300209522247314, |
| "eval_runtime": 1140.6996, |
| "eval_samples_per_second": 1.753, |
| "eval_steps_per_second": 0.877, |
| "step": 3800 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.946488568802324e-06, |
| "logits/chosen": -0.331181138753891, |
| "logits/rejected": 0.5133270025253296, |
| "logps/chosen": -477.48321533203125, |
| "logps/rejected": -547.2113647460938, |
| "loss": 0.507, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.1447014808654785, |
| "rewards/margins": 0.8755722045898438, |
| "rewards/rejected": -3.0202736854553223, |
| "step": 3810 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.935245276944278e-06, |
| "logits/chosen": -0.2571033537387848, |
| "logits/rejected": 0.4771188199520111, |
| "logps/chosen": -497.06866455078125, |
| "logps/rejected": -538.19091796875, |
| "loss": 0.5672, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.0118210315704346, |
| "rewards/margins": 0.7694553136825562, |
| "rewards/rejected": -2.781276226043701, |
| "step": 3820 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.9239928993249723e-06, |
| "logits/chosen": -0.2940518260002136, |
| "logits/rejected": 0.16310206055641174, |
| "logps/chosen": -479.2884216308594, |
| "logps/rejected": -555.8187255859375, |
| "loss": 0.4874, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.98129141330719, |
| "rewards/margins": 1.1137723922729492, |
| "rewards/rejected": -3.0950639247894287, |
| "step": 3830 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.912731670838207e-06, |
| "logits/chosen": -0.561539888381958, |
| "logits/rejected": 0.4365290701389313, |
| "logps/chosen": -458.83734130859375, |
| "logps/rejected": -545.0903930664062, |
| "loss": 0.5649, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.022507667541504, |
| "rewards/margins": 0.8180673718452454, |
| "rewards/rejected": -2.8405749797821045, |
| "step": 3840 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.901461826562543e-06, |
| "logits/chosen": -0.57462078332901, |
| "logits/rejected": 0.3850022852420807, |
| "logps/chosen": -398.58343505859375, |
| "logps/rejected": -475.1954040527344, |
| "loss": 0.4639, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.729619026184082, |
| "rewards/margins": 0.9505112767219543, |
| "rewards/rejected": -2.6801302433013916, |
| "step": 3850 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.8901836017563966e-06, |
| "logits/chosen": -0.42263826727867126, |
| "logits/rejected": 0.20525923371315002, |
| "logps/chosen": -463.990478515625, |
| "logps/rejected": -499.493408203125, |
| "loss": 0.562, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.830883264541626, |
| "rewards/margins": 0.7106839418411255, |
| "rewards/rejected": -2.541567325592041, |
| "step": 3860 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.8788972318531272e-06, |
| "logits/chosen": -0.6156303882598877, |
| "logits/rejected": 0.3582186996936798, |
| "logps/chosen": -427.96002197265625, |
| "logps/rejected": -484.9053649902344, |
| "loss": 0.5611, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.7909084558486938, |
| "rewards/margins": 0.6238812208175659, |
| "rewards/rejected": -2.4147896766662598, |
| "step": 3870 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.8676029524561255e-06, |
| "logits/chosen": -0.09342961758375168, |
| "logits/rejected": 0.25026214122772217, |
| "logps/chosen": -468.81072998046875, |
| "logps/rejected": -550.8553466796875, |
| "loss": 0.5211, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7681678533554077, |
| "rewards/margins": 0.9555566906929016, |
| "rewards/rejected": -2.723724603652954, |
| "step": 3880 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.8563009993338906e-06, |
| "logits/chosen": -0.24521970748901367, |
| "logits/rejected": 0.4679892659187317, |
| "logps/chosen": -440.3994140625, |
| "logps/rejected": -543.8873291015625, |
| "loss": 0.4792, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.9353783130645752, |
| "rewards/margins": 1.108598232269287, |
| "rewards/rejected": -3.043976306915283, |
| "step": 3890 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.844991608415113e-06, |
| "logits/chosen": -0.3554648458957672, |
| "logits/rejected": -0.03292546421289444, |
| "logps/chosen": -480.20037841796875, |
| "logps/rejected": -566.8743896484375, |
| "loss": 0.5193, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.146230936050415, |
| "rewards/margins": 0.9551491737365723, |
| "rewards/rejected": -3.1013801097869873, |
| "step": 3900 |
| }, |
| { |
| "epoch": 0.51, |
| "eval_logits/chosen": 1.3990401029586792, |
| "eval_logits/rejected": 2.1469078063964844, |
| "eval_logps/chosen": -485.6194152832031, |
| "eval_logps/rejected": -559.7595825195312, |
| "eval_loss": 0.5184990167617798, |
| "eval_rewards/accuracies": 0.734000027179718, |
| "eval_rewards/chosen": -2.1712939739227295, |
| "eval_rewards/margins": 0.940490186214447, |
| "eval_rewards/rejected": -3.111783981323242, |
| "eval_runtime": 1152.1911, |
| "eval_samples_per_second": 1.736, |
| "eval_steps_per_second": 0.868, |
| "step": 3900 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.833675015783746e-06, |
| "logits/chosen": -0.07204243540763855, |
| "logits/rejected": 0.11534376442432404, |
| "logps/chosen": -458.18798828125, |
| "logps/rejected": -554.1547241210938, |
| "loss": 0.5293, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.2303760051727295, |
| "rewards/margins": 0.9195043444633484, |
| "rewards/rejected": -3.1498801708221436, |
| "step": 3910 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.8223514576740784e-06, |
| "logits/chosen": 0.09607383608818054, |
| "logits/rejected": 0.03933734819293022, |
| "logps/chosen": -431.48175048828125, |
| "logps/rejected": -560.3643798828125, |
| "loss": 0.4971, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.0321695804595947, |
| "rewards/margins": 0.9992935061454773, |
| "rewards/rejected": -3.0314629077911377, |
| "step": 3920 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.8110211704658073e-06, |
| "logits/chosen": -0.698123037815094, |
| "logits/rejected": 0.5209210515022278, |
| "logps/chosen": -530.5778198242188, |
| "logps/rejected": -592.2348022460938, |
| "loss": 0.4693, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.271674156188965, |
| "rewards/margins": 0.9807974696159363, |
| "rewards/rejected": -3.252471446990967, |
| "step": 3930 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.7996843906790955e-06, |
| "logits/chosen": -0.11168187856674194, |
| "logits/rejected": 0.6565873026847839, |
| "logps/chosen": -470.85064697265625, |
| "logps/rejected": -538.1116943359375, |
| "loss": 0.6032, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.2667603492736816, |
| "rewards/margins": 0.637987494468689, |
| "rewards/rejected": -2.9047482013702393, |
| "step": 3940 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.7883413549696396e-06, |
| "logits/chosen": -0.47097617387771606, |
| "logits/rejected": 0.9360452890396118, |
| "logps/chosen": -505.99658203125, |
| "logps/rejected": -607.1832885742188, |
| "loss": 0.3929, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.171560764312744, |
| "rewards/margins": 1.2839739322662354, |
| "rewards/rejected": -3.4555351734161377, |
| "step": 3950 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.776992300123732e-06, |
| "logits/chosen": -0.3683644235134125, |
| "logits/rejected": 0.48894062638282776, |
| "logps/chosen": -468.18316650390625, |
| "logps/rejected": -581.08154296875, |
| "loss": 0.5471, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.261422872543335, |
| "rewards/margins": 1.1004810333251953, |
| "rewards/rejected": -3.3619041442871094, |
| "step": 3960 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.7656374630533113e-06, |
| "logits/chosen": -0.399179607629776, |
| "logits/rejected": -0.14117364585399628, |
| "logps/chosen": -440.6139221191406, |
| "logps/rejected": -555.7425537109375, |
| "loss": 0.4605, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.0904624462127686, |
| "rewards/margins": 1.1303551197052002, |
| "rewards/rejected": -3.2208175659179688, |
| "step": 3970 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.754277080791021e-06, |
| "logits/chosen": -0.6631547212600708, |
| "logits/rejected": -0.17448690533638, |
| "logps/chosen": -481.0576171875, |
| "logps/rejected": -556.90380859375, |
| "loss": 0.6197, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -2.1500160694122314, |
| "rewards/margins": 0.8200462460517883, |
| "rewards/rejected": -2.970062255859375, |
| "step": 3980 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.742911390485262e-06, |
| "logits/chosen": -0.10709667205810547, |
| "logits/rejected": 0.19126668572425842, |
| "logps/chosen": -419.6375427246094, |
| "logps/rejected": -470.33880615234375, |
| "loss": 0.6197, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.0977683067321777, |
| "rewards/margins": 0.590727686882019, |
| "rewards/rejected": -2.6884961128234863, |
| "step": 3990 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.731540629395239e-06, |
| "logits/chosen": -0.5709615349769592, |
| "logits/rejected": 0.3959980607032776, |
| "logps/chosen": -460.33544921875, |
| "logps/rejected": -499.603759765625, |
| "loss": 0.5764, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.8743162155151367, |
| "rewards/margins": 0.6547318696975708, |
| "rewards/rejected": -2.529047966003418, |
| "step": 4000 |
| }, |
| { |
| "epoch": 0.52, |
| "eval_logits/chosen": 1.1191996335983276, |
| "eval_logits/rejected": 1.8653334379196167, |
| "eval_logps/chosen": -469.05755615234375, |
| "eval_logps/rejected": -545.9298095703125, |
| "eval_loss": 0.5176907777786255, |
| "eval_rewards/accuracies": 0.7310000061988831, |
| "eval_rewards/chosen": -2.0056753158569336, |
| "eval_rewards/margins": 0.9678111672401428, |
| "eval_rewards/rejected": -2.9734864234924316, |
| "eval_runtime": 1155.1218, |
| "eval_samples_per_second": 1.731, |
| "eval_steps_per_second": 0.866, |
| "step": 4000 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.7201650348860115e-06, |
| "logits/chosen": -0.7722757458686829, |
| "logits/rejected": 0.36960023641586304, |
| "logps/chosen": -442.9876403808594, |
| "logps/rejected": -502.1902770996094, |
| "loss": 0.4848, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.029517412185669, |
| "rewards/margins": 1.0260822772979736, |
| "rewards/rejected": -3.0555994510650635, |
| "step": 4010 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.7087848444235354e-06, |
| "logits/chosen": -0.8600364923477173, |
| "logits/rejected": 0.4562684893608093, |
| "logps/chosen": -454.643310546875, |
| "logps/rejected": -560.4906005859375, |
| "loss": 0.4478, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.75933837890625, |
| "rewards/margins": 1.259271264076233, |
| "rewards/rejected": -3.0186095237731934, |
| "step": 4020 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.697400295569707e-06, |
| "logits/chosen": -0.6417149901390076, |
| "logits/rejected": -0.5029559135437012, |
| "logps/chosen": -410.6507873535156, |
| "logps/rejected": -496.6881408691406, |
| "loss": 0.5825, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.6875232458114624, |
| "rewards/margins": 0.8735893964767456, |
| "rewards/rejected": -2.561112880706787, |
| "step": 4030 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.6860116259774065e-06, |
| "logits/chosen": -0.604103684425354, |
| "logits/rejected": 0.4802830219268799, |
| "logps/chosen": -457.4012145996094, |
| "logps/rejected": -566.4182739257812, |
| "loss": 0.4229, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.7043393850326538, |
| "rewards/margins": 1.2353878021240234, |
| "rewards/rejected": -2.939727306365967, |
| "step": 4040 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.674619073385531e-06, |
| "logits/chosen": -0.3527432382106781, |
| "logits/rejected": 0.4966762959957123, |
| "logps/chosen": -414.24102783203125, |
| "logps/rejected": -521.317626953125, |
| "loss": 0.5281, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.7188953161239624, |
| "rewards/margins": 1.0378594398498535, |
| "rewards/rejected": -2.7567548751831055, |
| "step": 4050 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.663222875614038e-06, |
| "logits/chosen": -0.4794388711452484, |
| "logits/rejected": 0.13224102556705475, |
| "logps/chosen": -445.65087890625, |
| "logps/rejected": -518.9547729492188, |
| "loss": 0.6135, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.054771661758423, |
| "rewards/margins": 0.5872553586959839, |
| "rewards/rejected": -2.6420271396636963, |
| "step": 4060 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.6518232705589775e-06, |
| "logits/chosen": -0.4580906331539154, |
| "logits/rejected": 0.23300707340240479, |
| "logps/chosen": -428.31707763671875, |
| "logps/rejected": -548.2869262695312, |
| "loss": 0.4599, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.6607511043548584, |
| "rewards/margins": 1.2533749341964722, |
| "rewards/rejected": -2.91412615776062, |
| "step": 4070 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.640420496187528e-06, |
| "logits/chosen": -0.6754065752029419, |
| "logits/rejected": 0.6897384524345398, |
| "logps/chosen": -462.10235595703125, |
| "logps/rejected": -518.758544921875, |
| "loss": 0.4402, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -1.6582934856414795, |
| "rewards/margins": 1.126479983329773, |
| "rewards/rejected": -2.784773588180542, |
| "step": 4080 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.629014790533025e-06, |
| "logits/chosen": -0.8516901135444641, |
| "logits/rejected": 0.13358630239963531, |
| "logps/chosen": -460.47509765625, |
| "logps/rejected": -500.4847106933594, |
| "loss": 0.4939, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.6765422821044922, |
| "rewards/margins": 1.0363404750823975, |
| "rewards/rejected": -2.7128829956054688, |
| "step": 4090 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.617606391689996e-06, |
| "logits/chosen": -0.5585889220237732, |
| "logits/rejected": 0.3247618079185486, |
| "logps/chosen": -427.287841796875, |
| "logps/rejected": -505.0663146972656, |
| "loss": 0.504, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.6491613388061523, |
| "rewards/margins": 0.9884650111198425, |
| "rewards/rejected": -2.6376264095306396, |
| "step": 4100 |
| }, |
| { |
| "epoch": 0.54, |
| "eval_logits/chosen": 1.0344183444976807, |
| "eval_logits/rejected": 1.7948068380355835, |
| "eval_logps/chosen": -450.85650634765625, |
| "eval_logps/rejected": -523.1134643554688, |
| "eval_loss": 0.5179835557937622, |
| "eval_rewards/accuracies": 0.7269999980926514, |
| "eval_rewards/chosen": -1.8236651420593262, |
| "eval_rewards/margins": 0.9216578006744385, |
| "eval_rewards/rejected": -2.7453227043151855, |
| "eval_runtime": 1151.937, |
| "eval_samples_per_second": 1.736, |
| "eval_steps_per_second": 0.868, |
| "step": 4100 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.6061955378091896e-06, |
| "logits/chosen": -0.4794127345085144, |
| "logits/rejected": 0.3910555839538574, |
| "logps/chosen": -423.6997985839844, |
| "logps/rejected": -548.6406860351562, |
| "loss": 0.4475, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8037827014923096, |
| "rewards/margins": 1.150130271911621, |
| "rewards/rejected": -2.9539127349853516, |
| "step": 4110 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.5947824670926025e-06, |
| "logits/chosen": -0.4441138803958893, |
| "logits/rejected": -0.20847466588020325, |
| "logps/chosen": -416.5233459472656, |
| "logps/rejected": -513.5562744140625, |
| "loss": 0.5321, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.7394109964370728, |
| "rewards/margins": 0.8989079594612122, |
| "rewards/rejected": -2.6383190155029297, |
| "step": 4120 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.583367417788508e-06, |
| "logits/chosen": -0.3594195246696472, |
| "logits/rejected": 0.763751745223999, |
| "logps/chosen": -463.782470703125, |
| "logps/rejected": -538.66748046875, |
| "loss": 0.5754, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -2.1264193058013916, |
| "rewards/margins": 0.9109905958175659, |
| "rewards/rejected": -3.037409543991089, |
| "step": 4130 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.5719506281864838e-06, |
| "logits/chosen": -0.767003059387207, |
| "logits/rejected": -0.14061614871025085, |
| "logps/chosen": -479.3724670410156, |
| "logps/rejected": -501.5732421875, |
| "loss": 0.5855, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.9783029556274414, |
| "rewards/margins": 0.8176881670951843, |
| "rewards/rejected": -2.7959909439086914, |
| "step": 4140 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.5605323366124335e-06, |
| "logits/chosen": -0.4945451617240906, |
| "logits/rejected": 0.5269497036933899, |
| "logps/chosen": -454.4396057128906, |
| "logps/rejected": -526.3812866210938, |
| "loss": 0.513, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9973022937774658, |
| "rewards/margins": 0.7900811433792114, |
| "rewards/rejected": -2.787383556365967, |
| "step": 4150 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.5491127814236172e-06, |
| "logits/chosen": -0.19666481018066406, |
| "logits/rejected": -0.4852234423160553, |
| "logps/chosen": -372.50628662109375, |
| "logps/rejected": -494.17144775390625, |
| "loss": 0.51, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.6012966632843018, |
| "rewards/margins": 0.775696337223053, |
| "rewards/rejected": -2.37699294090271, |
| "step": 4160 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.537692201003671e-06, |
| "logits/chosen": -0.26586753129959106, |
| "logits/rejected": 0.2930835485458374, |
| "logps/chosen": -467.44317626953125, |
| "logps/rejected": -552.4528198242188, |
| "loss": 0.5688, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.0270495414733887, |
| "rewards/margins": 0.9594928622245789, |
| "rewards/rejected": -2.9865424633026123, |
| "step": 4170 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.526270833757635e-06, |
| "logits/chosen": -0.6537747383117676, |
| "logits/rejected": 0.6223368048667908, |
| "logps/chosen": -447.3667907714844, |
| "logps/rejected": -499.9156188964844, |
| "loss": 0.5537, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9054014682769775, |
| "rewards/margins": 0.7102811932563782, |
| "rewards/rejected": -2.615682601928711, |
| "step": 4180 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.514848918106971e-06, |
| "logits/chosen": -0.553719162940979, |
| "logits/rejected": 0.4931580424308777, |
| "logps/chosen": -447.16204833984375, |
| "logps/rejected": -520.3040771484375, |
| "loss": 0.4678, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.8360874652862549, |
| "rewards/margins": 1.1568195819854736, |
| "rewards/rejected": -2.9929070472717285, |
| "step": 4190 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.503426692484594e-06, |
| "logits/chosen": -0.3490946888923645, |
| "logits/rejected": 0.03165990114212036, |
| "logps/chosen": -444.204345703125, |
| "logps/rejected": -547.2092895507812, |
| "loss": 0.4846, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9349002838134766, |
| "rewards/margins": 0.9464532732963562, |
| "rewards/rejected": -2.8813538551330566, |
| "step": 4200 |
| }, |
| { |
| "epoch": 0.55, |
| "eval_logits/chosen": 1.3328680992126465, |
| "eval_logits/rejected": 2.1064412593841553, |
| "eval_logps/chosen": -480.6316833496094, |
| "eval_logps/rejected": -553.0634765625, |
| "eval_loss": 0.5167676210403442, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -2.1214165687561035, |
| "eval_rewards/margins": 0.9234069585800171, |
| "eval_rewards/rejected": -3.044823169708252, |
| "eval_runtime": 1156.9425, |
| "eval_samples_per_second": 1.729, |
| "eval_steps_per_second": 0.864, |
| "step": 4200 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.492004395329883e-06, |
| "logits/chosen": -0.47201377153396606, |
| "logits/rejected": 0.1662154495716095, |
| "logps/chosen": -442.28265380859375, |
| "logps/rejected": -555.5616455078125, |
| "loss": 0.424, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.9666084051132202, |
| "rewards/margins": 1.2794740200042725, |
| "rewards/rejected": -3.246082305908203, |
| "step": 4210 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.4805822650837165e-06, |
| "logits/chosen": -0.05408313125371933, |
| "logits/rejected": 0.7589906454086304, |
| "logps/chosen": -454.1454162597656, |
| "logps/rejected": -595.169189453125, |
| "loss": 0.4313, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.1684341430664062, |
| "rewards/margins": 1.3142242431640625, |
| "rewards/rejected": -3.4826583862304688, |
| "step": 4220 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.4691605401834843e-06, |
| "logits/chosen": -0.5830433964729309, |
| "logits/rejected": 0.2723081707954407, |
| "logps/chosen": -507.00323486328125, |
| "logps/rejected": -582.15283203125, |
| "loss": 0.5321, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.2436351776123047, |
| "rewards/margins": 0.8307396173477173, |
| "rewards/rejected": -3.0743746757507324, |
| "step": 4230 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.457739459058117e-06, |
| "logits/chosen": -0.5049376487731934, |
| "logits/rejected": 0.14484481513500214, |
| "logps/chosen": -547.284423828125, |
| "logps/rejected": -602.6152954101562, |
| "loss": 0.4511, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.2237820625305176, |
| "rewards/margins": 1.0236866474151611, |
| "rewards/rejected": -3.2474684715270996, |
| "step": 4240 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.4463192601231054e-06, |
| "logits/chosen": -0.28555774688720703, |
| "logits/rejected": 0.8731438517570496, |
| "logps/chosen": -530.2666015625, |
| "logps/rejected": -578.294677734375, |
| "loss": 0.4807, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.338956117630005, |
| "rewards/margins": 1.1115846633911133, |
| "rewards/rejected": -3.450540542602539, |
| "step": 4250 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.434900181775524e-06, |
| "logits/chosen": -0.7155624628067017, |
| "logits/rejected": 0.29977577924728394, |
| "logps/chosen": -479.2982482910156, |
| "logps/rejected": -573.4529418945312, |
| "loss": 0.4766, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.0659286975860596, |
| "rewards/margins": 1.073961615562439, |
| "rewards/rejected": -3.139890193939209, |
| "step": 4260 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.4234824623890578e-06, |
| "logits/chosen": -0.7157832980155945, |
| "logits/rejected": 0.18825335800647736, |
| "logps/chosen": -466.107177734375, |
| "logps/rejected": -545.6228637695312, |
| "loss": 0.4646, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.0345892906188965, |
| "rewards/margins": 0.9853957295417786, |
| "rewards/rejected": -3.0199851989746094, |
| "step": 4270 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.4120663403090193e-06, |
| "logits/chosen": -0.6923838257789612, |
| "logits/rejected": 0.20935389399528503, |
| "logps/chosen": -482.73724365234375, |
| "logps/rejected": -583.6288452148438, |
| "loss": 0.5585, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.0791468620300293, |
| "rewards/margins": 0.9277111291885376, |
| "rewards/rejected": -3.0068578720092773, |
| "step": 4280 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.40065205384738e-06, |
| "logits/chosen": -0.7260463833808899, |
| "logits/rejected": 0.5240879058837891, |
| "logps/chosen": -472.66845703125, |
| "logps/rejected": -497.08575439453125, |
| "loss": 0.6256, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.3160202503204346, |
| "rewards/margins": 0.5535823106765747, |
| "rewards/rejected": -2.8696022033691406, |
| "step": 4290 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.389239841277793e-06, |
| "logits/chosen": -0.25377124547958374, |
| "logits/rejected": 0.46026620268821716, |
| "logps/chosen": -445.01019287109375, |
| "logps/rejected": -527.544677734375, |
| "loss": 0.426, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.9602826833724976, |
| "rewards/margins": 1.1500470638275146, |
| "rewards/rejected": -3.1103296279907227, |
| "step": 4300 |
| }, |
| { |
| "epoch": 0.56, |
| "eval_logits/chosen": 1.2899590730667114, |
| "eval_logits/rejected": 2.0376882553100586, |
| "eval_logps/chosen": -469.9074401855469, |
| "eval_logps/rejected": -543.4855346679688, |
| "eval_loss": 0.5095502138137817, |
| "eval_rewards/accuracies": 0.7325000166893005, |
| "eval_rewards/chosen": -2.014173746109009, |
| "eval_rewards/margins": 0.9348700642585754, |
| "eval_rewards/rejected": -2.9490435123443604, |
| "eval_runtime": 1156.5138, |
| "eval_samples_per_second": 1.729, |
| "eval_steps_per_second": 0.865, |
| "step": 4300 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.3778299408306167e-06, |
| "logits/chosen": -0.4929943084716797, |
| "logits/rejected": 0.48326101899147034, |
| "logps/chosen": -455.02838134765625, |
| "logps/rejected": -525.9819946289062, |
| "loss": 0.5103, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.0632247924804688, |
| "rewards/margins": 0.8879944682121277, |
| "rewards/rejected": -2.951219081878662, |
| "step": 4310 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.3664225906879452e-06, |
| "logits/chosen": -0.42962485551834106, |
| "logits/rejected": 0.17619100213050842, |
| "logps/chosen": -452.4551696777344, |
| "logps/rejected": -521.5052490234375, |
| "loss": 0.4912, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.080214023590088, |
| "rewards/margins": 0.9484179615974426, |
| "rewards/rejected": -3.0286319255828857, |
| "step": 4320 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.3550180289786357e-06, |
| "logits/chosen": -0.8907386064529419, |
| "logits/rejected": 0.4001527726650238, |
| "logps/chosen": -469.4991760253906, |
| "logps/rejected": -503.5176696777344, |
| "loss": 0.5744, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -2.0900187492370605, |
| "rewards/margins": 0.7551154494285583, |
| "rewards/rejected": -2.8451342582702637, |
| "step": 4330 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.343616493773335e-06, |
| "logits/chosen": -0.3753073513507843, |
| "logits/rejected": -0.04642016813158989, |
| "logps/chosen": -479.0115661621094, |
| "logps/rejected": -580.8361206054688, |
| "loss": 0.5041, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.154541492462158, |
| "rewards/margins": 1.0236194133758545, |
| "rewards/rejected": -3.1781609058380127, |
| "step": 4340 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.3322182230795127e-06, |
| "logits/chosen": -0.14095106720924377, |
| "logits/rejected": -0.2498963177204132, |
| "logps/chosen": -418.10491943359375, |
| "logps/rejected": -577.77783203125, |
| "loss": 0.4198, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.8447338342666626, |
| "rewards/margins": 1.3301194906234741, |
| "rewards/rejected": -3.1748533248901367, |
| "step": 4350 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.320823454836491e-06, |
| "logits/chosen": -0.9373834729194641, |
| "logits/rejected": 0.3088436722755432, |
| "logps/chosen": -433.88226318359375, |
| "logps/rejected": -530.290283203125, |
| "loss": 0.401, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.7864519357681274, |
| "rewards/margins": 1.156484603881836, |
| "rewards/rejected": -2.942936658859253, |
| "step": 4360 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.309432426910478e-06, |
| "logits/chosen": -0.6841565370559692, |
| "logits/rejected": 0.7033779621124268, |
| "logps/chosen": -494.7813415527344, |
| "logps/rejected": -543.1317138671875, |
| "loss": 0.4644, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.9815502166748047, |
| "rewards/margins": 1.0772145986557007, |
| "rewards/rejected": -3.058764934539795, |
| "step": 4370 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.298045377089604e-06, |
| "logits/chosen": -0.8424911499023438, |
| "logits/rejected": 0.32612770795822144, |
| "logps/chosen": -466.86895751953125, |
| "logps/rejected": -569.2764892578125, |
| "loss": 0.4831, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.1209797859191895, |
| "rewards/margins": 1.2162678241729736, |
| "rewards/rejected": -3.337247371673584, |
| "step": 4380 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.286662543078955e-06, |
| "logits/chosen": -0.4193035066127777, |
| "logits/rejected": 0.4950118958950043, |
| "logps/chosen": -491.828857421875, |
| "logps/rejected": -533.1112060546875, |
| "loss": 0.5354, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.155092239379883, |
| "rewards/margins": 0.7993155121803284, |
| "rewards/rejected": -2.9544079303741455, |
| "step": 4390 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.2752841624956125e-06, |
| "logits/chosen": -0.604637622833252, |
| "logits/rejected": 0.18484275043010712, |
| "logps/chosen": -507.84112548828125, |
| "logps/rejected": -585.2672119140625, |
| "loss": 0.5289, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.1505980491638184, |
| "rewards/margins": 0.9751760363578796, |
| "rewards/rejected": -3.1257741451263428, |
| "step": 4400 |
| }, |
| { |
| "epoch": 0.58, |
| "eval_logits/chosen": 1.0285941362380981, |
| "eval_logits/rejected": 1.7669211626052856, |
| "eval_logps/chosen": -464.733154296875, |
| "eval_logps/rejected": -542.265869140625, |
| "eval_loss": 0.5142533779144287, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -1.9624314308166504, |
| "eval_rewards/margins": 0.9744157195091248, |
| "eval_rewards/rejected": -2.93684720993042, |
| "eval_runtime": 1341.7741, |
| "eval_samples_per_second": 1.491, |
| "eval_steps_per_second": 0.745, |
| "step": 4400 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.2639104728636915e-06, |
| "logits/chosen": -0.3302585482597351, |
| "logits/rejected": -0.0848245620727539, |
| "logps/chosen": -438.95501708984375, |
| "logps/rejected": -518.0885009765625, |
| "loss": 0.5619, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7836341857910156, |
| "rewards/margins": 0.8088982701301575, |
| "rewards/rejected": -2.5925326347351074, |
| "step": 4410 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.252541711609384e-06, |
| "logits/chosen": -0.4086834788322449, |
| "logits/rejected": 0.6533911824226379, |
| "logps/chosen": -442.6697692871094, |
| "logps/rejected": -500.9053649902344, |
| "loss": 0.5224, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.846428632736206, |
| "rewards/margins": 0.9241348505020142, |
| "rewards/rejected": -2.7705636024475098, |
| "step": 4420 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.241178116056002e-06, |
| "logits/chosen": -0.7984479665756226, |
| "logits/rejected": 0.013322305865585804, |
| "logps/chosen": -431.56884765625, |
| "logps/rejected": -503.01080322265625, |
| "loss": 0.4753, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7908424139022827, |
| "rewards/margins": 0.9538822174072266, |
| "rewards/rejected": -2.744724750518799, |
| "step": 4430 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.2298199234190236e-06, |
| "logits/chosen": -0.3780064582824707, |
| "logits/rejected": 0.18104317784309387, |
| "logps/chosen": -475.06024169921875, |
| "logps/rejected": -548.1165771484375, |
| "loss": 0.4519, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.868931770324707, |
| "rewards/margins": 1.0790822505950928, |
| "rewards/rejected": -2.9480140209198, |
| "step": 4440 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.218467370801138e-06, |
| "logits/chosen": -0.6153514981269836, |
| "logits/rejected": 0.3399887979030609, |
| "logps/chosen": -482.9944763183594, |
| "logps/rejected": -523.6447143554688, |
| "loss": 0.6105, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.1587672233581543, |
| "rewards/margins": 0.7031978368759155, |
| "rewards/rejected": -2.8619649410247803, |
| "step": 4450 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.207120695187304e-06, |
| "logits/chosen": -0.7532152533531189, |
| "logits/rejected": 0.8358518481254578, |
| "logps/chosen": -490.2608337402344, |
| "logps/rejected": -566.9043579101562, |
| "loss": 0.4384, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.088639736175537, |
| "rewards/margins": 1.1486093997955322, |
| "rewards/rejected": -3.2372488975524902, |
| "step": 4460 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.195780133439794e-06, |
| "logits/chosen": -0.20792141556739807, |
| "logits/rejected": 0.016714613884687424, |
| "logps/chosen": -480.798583984375, |
| "logps/rejected": -579.108642578125, |
| "loss": 0.5193, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.015219211578369, |
| "rewards/margins": 0.9281864166259766, |
| "rewards/rejected": -2.9434056282043457, |
| "step": 4470 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.1844459222932535e-06, |
| "logits/chosen": -0.6897789835929871, |
| "logits/rejected": 0.44204026460647583, |
| "logps/chosen": -500.740478515625, |
| "logps/rejected": -560.5718383789062, |
| "loss": 0.4913, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.2337794303894043, |
| "rewards/margins": 0.9274255633354187, |
| "rewards/rejected": -3.161205768585205, |
| "step": 4480 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.17311829834976e-06, |
| "logits/chosen": -1.0006176233291626, |
| "logits/rejected": -0.1672603338956833, |
| "logps/chosen": -442.28802490234375, |
| "logps/rejected": -546.0424194335938, |
| "loss": 0.4748, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.864127516746521, |
| "rewards/margins": 1.0716313123703003, |
| "rewards/rejected": -2.9357590675354004, |
| "step": 4490 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.1617974980738814e-06, |
| "logits/chosen": -0.7605545520782471, |
| "logits/rejected": 0.6477273106575012, |
| "logps/chosen": -446.9376525878906, |
| "logps/rejected": -515.064453125, |
| "loss": 0.4542, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9530082941055298, |
| "rewards/margins": 0.973568320274353, |
| "rewards/rejected": -2.926576852798462, |
| "step": 4500 |
| }, |
| { |
| "epoch": 0.59, |
| "eval_logits/chosen": 1.1394939422607422, |
| "eval_logits/rejected": 1.8774791955947876, |
| "eval_logps/chosen": -464.9223327636719, |
| "eval_logps/rejected": -541.3861083984375, |
| "eval_loss": 0.5101990699768066, |
| "eval_rewards/accuracies": 0.7335000038146973, |
| "eval_rewards/chosen": -1.9643235206604004, |
| "eval_rewards/margins": 0.963726282119751, |
| "eval_rewards/rejected": -2.9280498027801514, |
| "eval_runtime": 1305.9412, |
| "eval_samples_per_second": 1.531, |
| "eval_steps_per_second": 0.766, |
| "step": 4500 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.150483757787744e-06, |
| "logits/chosen": -0.9358876943588257, |
| "logits/rejected": 0.6876541972160339, |
| "logps/chosen": -459.2244567871094, |
| "logps/rejected": -489.6673278808594, |
| "loss": 0.5597, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.086225986480713, |
| "rewards/margins": 0.8154786229133606, |
| "rewards/rejected": -2.9017043113708496, |
| "step": 4510 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.139177313666093e-06, |
| "logits/chosen": -0.5859389901161194, |
| "logits/rejected": -0.10606422275304794, |
| "logps/chosen": -503.25933837890625, |
| "logps/rejected": -543.6451416015625, |
| "loss": 0.533, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.0373759269714355, |
| "rewards/margins": 0.933161735534668, |
| "rewards/rejected": -2.9705376625061035, |
| "step": 4520 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.1278784017313688e-06, |
| "logits/chosen": -0.2547047734260559, |
| "logits/rejected": -0.3063717484474182, |
| "logps/chosen": -468.4463806152344, |
| "logps/rejected": -556.9096069335938, |
| "loss": 0.5121, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.8143895864486694, |
| "rewards/margins": 0.8284521102905273, |
| "rewards/rejected": -2.6428415775299072, |
| "step": 4530 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.116587257848776e-06, |
| "logits/chosen": -0.6765443086624146, |
| "logits/rejected": -0.6112786531448364, |
| "logps/chosen": -419.4952697753906, |
| "logps/rejected": -512.7720947265625, |
| "loss": 0.587, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.773984670639038, |
| "rewards/margins": 0.6838240623474121, |
| "rewards/rejected": -2.4578089714050293, |
| "step": 4540 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.105304117721361e-06, |
| "logits/chosen": -0.5224363803863525, |
| "logits/rejected": 0.21402570605278015, |
| "logps/chosen": -394.37713623046875, |
| "logps/rejected": -444.3313903808594, |
| "loss": 0.595, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.7446057796478271, |
| "rewards/margins": 0.7791751027107239, |
| "rewards/rejected": -2.5237812995910645, |
| "step": 4550 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.0940292168850913e-06, |
| "logits/chosen": -0.5463732481002808, |
| "logits/rejected": 0.1663360893726349, |
| "logps/chosen": -414.0874938964844, |
| "logps/rejected": -459.6375427246094, |
| "loss": 0.5231, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.5902382135391235, |
| "rewards/margins": 0.7520595788955688, |
| "rewards/rejected": -2.3422977924346924, |
| "step": 4560 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.082762790703939e-06, |
| "logits/chosen": -0.802535891532898, |
| "logits/rejected": 0.28110548853874207, |
| "logps/chosen": -429.9853515625, |
| "logps/rejected": -507.537841796875, |
| "loss": 0.5194, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.6454626321792603, |
| "rewards/margins": 0.8513976335525513, |
| "rewards/rejected": -2.4968602657318115, |
| "step": 4570 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.0715050743649674e-06, |
| "logits/chosen": -0.736199676990509, |
| "logits/rejected": 0.028460631147027016, |
| "logps/chosen": -406.27423095703125, |
| "logps/rejected": -543.2493286132812, |
| "loss": 0.4913, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.7144979238510132, |
| "rewards/margins": 0.9728431701660156, |
| "rewards/rejected": -2.6873412132263184, |
| "step": 4580 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.060256302873421e-06, |
| "logits/chosen": -0.4676334857940674, |
| "logits/rejected": -0.17427358031272888, |
| "logps/chosen": -442.82806396484375, |
| "logps/rejected": -547.2783203125, |
| "loss": 0.4903, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.9163872003555298, |
| "rewards/margins": 0.9291712641716003, |
| "rewards/rejected": -2.8455584049224854, |
| "step": 4590 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.049016711047822e-06, |
| "logits/chosen": -0.7500999569892883, |
| "logits/rejected": 0.41036081314086914, |
| "logps/chosen": -462.0934143066406, |
| "logps/rejected": -542.0261840820312, |
| "loss": 0.4839, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.009445905685425, |
| "rewards/margins": 1.0364575386047363, |
| "rewards/rejected": -3.045903444290161, |
| "step": 4600 |
| }, |
| { |
| "epoch": 0.6, |
| "eval_logits/chosen": 1.1471757888793945, |
| "eval_logits/rejected": 1.885801911354065, |
| "eval_logps/chosen": -468.8564147949219, |
| "eval_logps/rejected": -546.4149780273438, |
| "eval_loss": 0.5094394087791443, |
| "eval_rewards/accuracies": 0.7304999828338623, |
| "eval_rewards/chosen": -2.003664016723633, |
| "eval_rewards/margins": 0.9746747016906738, |
| "eval_rewards/rejected": -2.9783387184143066, |
| "eval_runtime": 1172.0125, |
| "eval_samples_per_second": 1.706, |
| "eval_steps_per_second": 0.853, |
| "step": 4600 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.037786533515064e-06, |
| "logits/chosen": -0.33482661843299866, |
| "logits/rejected": 0.0835098996758461, |
| "logps/chosen": -504.9192810058594, |
| "logps/rejected": -563.8491821289062, |
| "loss": 0.5111, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.06898832321167, |
| "rewards/margins": 0.8450971841812134, |
| "rewards/rejected": -2.914085626602173, |
| "step": 4610 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.02656600470552e-06, |
| "logits/chosen": -0.6622999906539917, |
| "logits/rejected": 0.010640504769980907, |
| "logps/chosen": -441.935546875, |
| "logps/rejected": -526.4178466796875, |
| "loss": 0.4816, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.8346233367919922, |
| "rewards/margins": 1.0247472524642944, |
| "rewards/rejected": -2.859370708465576, |
| "step": 4620 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 2.015355358848144e-06, |
| "logits/chosen": -0.013636887073516846, |
| "logits/rejected": -0.028725851327180862, |
| "logps/chosen": -397.56976318359375, |
| "logps/rejected": -514.6229248046875, |
| "loss": 0.5147, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.8054370880126953, |
| "rewards/margins": 0.9198155403137207, |
| "rewards/rejected": -2.725252866744995, |
| "step": 4630 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 2.004154829965582e-06, |
| "logits/chosen": -0.6800112128257751, |
| "logits/rejected": 0.04380284622311592, |
| "logps/chosen": -467.8673400878906, |
| "logps/rejected": -557.4893798828125, |
| "loss": 0.5006, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9010484218597412, |
| "rewards/margins": 1.0638830661773682, |
| "rewards/rejected": -2.9649314880371094, |
| "step": 4640 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 1.99296465186929e-06, |
| "logits/chosen": -0.8370498418807983, |
| "logits/rejected": 0.4450223445892334, |
| "logps/chosen": -462.6966247558594, |
| "logps/rejected": -493.6236877441406, |
| "loss": 0.514, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.8274908065795898, |
| "rewards/margins": 0.9097514152526855, |
| "rewards/rejected": -2.7372422218322754, |
| "step": 4650 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 1.9817850581546488e-06, |
| "logits/chosen": -0.24383525550365448, |
| "logits/rejected": 0.3440066874027252, |
| "logps/chosen": -495.785400390625, |
| "logps/rejected": -580.841796875, |
| "loss": 0.5974, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.167267084121704, |
| "rewards/margins": 0.7816963791847229, |
| "rewards/rejected": -2.9489636421203613, |
| "step": 4660 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 1.970616282196091e-06, |
| "logits/chosen": -0.5986989140510559, |
| "logits/rejected": 0.10017760097980499, |
| "logps/chosen": -437.3359375, |
| "logps/rejected": -534.2577514648438, |
| "loss": 0.4824, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.8744062185287476, |
| "rewards/margins": 1.0383882522583008, |
| "rewards/rejected": -2.912794589996338, |
| "step": 4670 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 1.959458557142228e-06, |
| "logits/chosen": -0.3310568928718567, |
| "logits/rejected": -0.00010063648369396105, |
| "logps/chosen": -468.37921142578125, |
| "logps/rejected": -538.8629150390625, |
| "loss": 0.7126, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -2.1639952659606934, |
| "rewards/margins": 0.5907930135726929, |
| "rewards/rejected": -2.754788398742676, |
| "step": 4680 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 1.948312115910982e-06, |
| "logits/chosen": -0.44464436173439026, |
| "logits/rejected": 0.15500059723854065, |
| "logps/chosen": -484.95843505859375, |
| "logps/rejected": -547.2438354492188, |
| "loss": 0.4925, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -2.0217325687408447, |
| "rewards/margins": 1.021672010421753, |
| "rewards/rejected": -3.0434045791625977, |
| "step": 4690 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.937177191184729e-06, |
| "logits/chosen": -0.2106432020664215, |
| "logits/rejected": -0.05869419500231743, |
| "logps/chosen": -432.0470275878906, |
| "logps/rejected": -516.4940185546875, |
| "loss": 0.5562, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.9173109531402588, |
| "rewards/margins": 0.8195775747299194, |
| "rewards/rejected": -2.7368886470794678, |
| "step": 4700 |
| }, |
| { |
| "epoch": 0.62, |
| "eval_logits/chosen": 1.1998752355575562, |
| "eval_logits/rejected": 1.9384359121322632, |
| "eval_logps/chosen": -471.0873107910156, |
| "eval_logps/rejected": -546.7677001953125, |
| "eval_loss": 0.5075832009315491, |
| "eval_rewards/accuracies": 0.734000027179718, |
| "eval_rewards/chosen": -2.025972843170166, |
| "eval_rewards/margins": 0.9558923244476318, |
| "eval_rewards/rejected": -2.9818649291992188, |
| "eval_runtime": 1220.8498, |
| "eval_samples_per_second": 1.638, |
| "eval_steps_per_second": 0.819, |
| "step": 4700 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.9260540154054317e-06, |
| "logits/chosen": -0.3351259231567383, |
| "logits/rejected": 0.35000115633010864, |
| "logps/chosen": -423.49017333984375, |
| "logps/rejected": -554.1179809570312, |
| "loss": 0.3844, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -1.8695085048675537, |
| "rewards/margins": 1.3363901376724243, |
| "rewards/rejected": -3.2058990001678467, |
| "step": 4710 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.9149428207697983e-06, |
| "logits/chosen": -0.33344680070877075, |
| "logits/rejected": -0.03340202569961548, |
| "logps/chosen": -488.39239501953125, |
| "logps/rejected": -548.125244140625, |
| "loss": 0.6408, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.3214313983917236, |
| "rewards/margins": 0.5961470603942871, |
| "rewards/rejected": -2.91757869720459, |
| "step": 4720 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.9038438392244262e-06, |
| "logits/chosen": -0.3149067759513855, |
| "logits/rejected": -0.3603346645832062, |
| "logps/chosen": -476.91766357421875, |
| "logps/rejected": -549.4235229492188, |
| "loss": 0.4499, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.9402230978012085, |
| "rewards/margins": 1.0026373863220215, |
| "rewards/rejected": -2.9428603649139404, |
| "step": 4730 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.8927573024609666e-06, |
| "logits/chosen": -0.267407089471817, |
| "logits/rejected": 0.3591880202293396, |
| "logps/chosen": -424.043212890625, |
| "logps/rejected": -519.4016723632812, |
| "loss": 0.4639, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -2.020537853240967, |
| "rewards/margins": 1.063114047050476, |
| "rewards/rejected": -3.0836517810821533, |
| "step": 4740 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.8816834419112845e-06, |
| "logits/chosen": -0.276968777179718, |
| "logits/rejected": 0.6711007356643677, |
| "logps/chosen": -467.92401123046875, |
| "logps/rejected": -534.5933837890625, |
| "loss": 0.5329, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.1998984813690186, |
| "rewards/margins": 1.0478665828704834, |
| "rewards/rejected": -3.247765302658081, |
| "step": 4750 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.8706224887426283e-06, |
| "logits/chosen": -0.017868299037218094, |
| "logits/rejected": 0.1659470647573471, |
| "logps/chosen": -508.35198974609375, |
| "logps/rejected": -592.7130126953125, |
| "loss": 0.5827, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.4431605339050293, |
| "rewards/margins": 0.7835296392440796, |
| "rewards/rejected": -3.2266902923583984, |
| "step": 4760 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 1.8595746738528045e-06, |
| "logits/chosen": -0.05624357610940933, |
| "logits/rejected": -0.07322041690349579, |
| "logps/chosen": -468.21783447265625, |
| "logps/rejected": -583.8419799804688, |
| "loss": 0.5432, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.211426258087158, |
| "rewards/margins": 0.9063574075698853, |
| "rewards/rejected": -3.117783546447754, |
| "step": 4770 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.8485402278653584e-06, |
| "logits/chosen": -0.331932932138443, |
| "logits/rejected": 0.19877803325653076, |
| "logps/chosen": -465.69122314453125, |
| "logps/rejected": -539.7578125, |
| "loss": 0.4492, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.2374541759490967, |
| "rewards/margins": 0.9785755276679993, |
| "rewards/rejected": -3.216029644012451, |
| "step": 4780 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.8375193811247577e-06, |
| "logits/chosen": -0.3239549398422241, |
| "logits/rejected": 0.33996933698654175, |
| "logps/chosen": -472.1409606933594, |
| "logps/rejected": -542.0296020507812, |
| "loss": 0.5019, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.24760365486145, |
| "rewards/margins": 0.8752537965774536, |
| "rewards/rejected": -3.1228575706481934, |
| "step": 4790 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.826512363691586e-06, |
| "logits/chosen": -0.8161247372627258, |
| "logits/rejected": -0.2031267136335373, |
| "logps/chosen": -500.17401123046875, |
| "logps/rejected": -567.0885009765625, |
| "loss": 0.4964, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.2298948764801025, |
| "rewards/margins": 0.9767163991928101, |
| "rewards/rejected": -3.206610918045044, |
| "step": 4800 |
| }, |
| { |
| "epoch": 0.63, |
| "eval_logits/chosen": 1.3968160152435303, |
| "eval_logits/rejected": 2.1537721157073975, |
| "eval_logps/chosen": -485.73052978515625, |
| "eval_logps/rejected": -561.4290161132812, |
| "eval_loss": 0.5078465938568115, |
| "eval_rewards/accuracies": 0.7335000038146973, |
| "eval_rewards/chosen": -2.172405481338501, |
| "eval_rewards/margins": 0.956072986125946, |
| "eval_rewards/rejected": -3.1284782886505127, |
| "eval_runtime": 1205.6317, |
| "eval_samples_per_second": 1.659, |
| "eval_steps_per_second": 0.829, |
| "step": 4800 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.8155194053377391e-06, |
| "logits/chosen": -0.4702053964138031, |
| "logits/rejected": 0.691516101360321, |
| "logps/chosen": -462.8230895996094, |
| "logps/rejected": -549.562744140625, |
| "loss": 0.478, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.0018343925476074, |
| "rewards/margins": 1.2428886890411377, |
| "rewards/rejected": -3.244723081588745, |
| "step": 4810 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.80454073554163e-06, |
| "logits/chosen": -0.25434356927871704, |
| "logits/rejected": 0.4281914234161377, |
| "logps/chosen": -443.32781982421875, |
| "logps/rejected": -518.9435424804688, |
| "loss": 0.5968, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.2195210456848145, |
| "rewards/margins": 0.8934110403060913, |
| "rewards/rejected": -3.1129322052001953, |
| "step": 4820 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.7935765834833966e-06, |
| "logits/chosen": -0.17489385604858398, |
| "logits/rejected": 0.3458006978034973, |
| "logps/chosen": -487.3931579589844, |
| "logps/rejected": -616.27587890625, |
| "loss": 0.4374, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.1858935356140137, |
| "rewards/margins": 1.282659888267517, |
| "rewards/rejected": -3.468553066253662, |
| "step": 4830 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.7826271780401182e-06, |
| "logits/chosen": -0.34163016080856323, |
| "logits/rejected": 0.6715383529663086, |
| "logps/chosen": -477.7323303222656, |
| "logps/rejected": -576.5382690429688, |
| "loss": 0.4413, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.3642547130584717, |
| "rewards/margins": 1.1457160711288452, |
| "rewards/rejected": -3.5099709033966064, |
| "step": 4840 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.7716927477810389e-06, |
| "logits/chosen": -0.3121749758720398, |
| "logits/rejected": 0.46080583333969116, |
| "logps/chosen": -460.3019104003906, |
| "logps/rejected": -576.9380493164062, |
| "loss": 0.4645, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -2.031574249267578, |
| "rewards/margins": 1.1349817514419556, |
| "rewards/rejected": -3.166555881500244, |
| "step": 4850 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.7607735209627953e-06, |
| "logits/chosen": -0.23650212585926056, |
| "logits/rejected": 0.6278365850448608, |
| "logps/chosen": -487.94305419921875, |
| "logps/rejected": -562.5647583007812, |
| "loss": 0.4918, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.249572515487671, |
| "rewards/margins": 1.1094893217086792, |
| "rewards/rejected": -3.3590621948242188, |
| "step": 4860 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.749869725524651e-06, |
| "logits/chosen": -0.2613787055015564, |
| "logits/rejected": 0.479464054107666, |
| "logps/chosen": -465.05743408203125, |
| "logps/rejected": -545.4020385742188, |
| "loss": 0.4567, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.889392614364624, |
| "rewards/margins": 1.1381362676620483, |
| "rewards/rejected": -3.027529001235962, |
| "step": 4870 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.7389815890837392e-06, |
| "logits/chosen": 0.03331397473812103, |
| "logits/rejected": -0.37613219022750854, |
| "logps/chosen": -484.1585998535156, |
| "logps/rejected": -606.52978515625, |
| "loss": 0.4609, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.071956157684326, |
| "rewards/margins": 1.0469639301300049, |
| "rewards/rejected": -3.118920087814331, |
| "step": 4880 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.7281093389303105e-06, |
| "logits/chosen": -0.5912939310073853, |
| "logits/rejected": -0.04338790848851204, |
| "logps/chosen": -431.63238525390625, |
| "logps/rejected": -512.7877197265625, |
| "loss": 0.4884, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8607794046401978, |
| "rewards/margins": 0.9917821884155273, |
| "rewards/rejected": -2.8525617122650146, |
| "step": 4890 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.7172532020229899e-06, |
| "logits/chosen": -0.8086379766464233, |
| "logits/rejected": 0.37625136971473694, |
| "logps/chosen": -488.94122314453125, |
| "logps/rejected": -558.6923828125, |
| "loss": 0.4879, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.099210739135742, |
| "rewards/margins": 1.0362012386322021, |
| "rewards/rejected": -3.1354117393493652, |
| "step": 4900 |
| }, |
| { |
| "epoch": 0.64, |
| "eval_logits/chosen": 1.3801579475402832, |
| "eval_logits/rejected": 2.132445812225342, |
| "eval_logps/chosen": -489.562255859375, |
| "eval_logps/rejected": -571.5598754882812, |
| "eval_loss": 0.5124561190605164, |
| "eval_rewards/accuracies": 0.7310000061988831, |
| "eval_rewards/chosen": -2.210721731185913, |
| "eval_rewards/margins": 1.0190653800964355, |
| "eval_rewards/rejected": -3.2297873497009277, |
| "eval_runtime": 1238.0078, |
| "eval_samples_per_second": 1.615, |
| "eval_steps_per_second": 0.808, |
| "step": 4900 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.7064134049840359e-06, |
| "logits/chosen": -0.2153107225894928, |
| "logits/rejected": 0.029362941160798073, |
| "logps/chosen": -449.80029296875, |
| "logps/rejected": -585.2144775390625, |
| "loss": 0.4087, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.9706871509552002, |
| "rewards/margins": 1.3460237979888916, |
| "rewards/rejected": -3.31671142578125, |
| "step": 4910 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 1.6955901740946136e-06, |
| "logits/chosen": -0.38802531361579895, |
| "logits/rejected": 0.2870200276374817, |
| "logps/chosen": -543.64306640625, |
| "logps/rejected": -657.2401123046875, |
| "loss": 0.483, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.5144500732421875, |
| "rewards/margins": 1.142197847366333, |
| "rewards/rejected": -3.6566474437713623, |
| "step": 4920 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.684783735290067e-06, |
| "logits/chosen": -0.6487330198287964, |
| "logits/rejected": 0.3835352659225464, |
| "logps/chosen": -452.5752868652344, |
| "logps/rejected": -591.7291870117188, |
| "loss": 0.3777, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.9951683282852173, |
| "rewards/margins": 1.4564054012298584, |
| "rewards/rejected": -3.451573610305786, |
| "step": 4930 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.6739943141552079e-06, |
| "logits/chosen": -0.6588994860649109, |
| "logits/rejected": 0.3580857515335083, |
| "logps/chosen": -522.0641479492188, |
| "logps/rejected": -571.3804931640625, |
| "loss": 0.5866, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.2700130939483643, |
| "rewards/margins": 1.0017244815826416, |
| "rewards/rejected": -3.271738052368164, |
| "step": 4940 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.663222135919601e-06, |
| "logits/chosen": -0.5175567865371704, |
| "logits/rejected": 0.5185331106185913, |
| "logps/chosen": -509.4527282714844, |
| "logps/rejected": -577.5142822265625, |
| "loss": 0.4903, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.127331256866455, |
| "rewards/margins": 0.9136832356452942, |
| "rewards/rejected": -3.0410146713256836, |
| "step": 4950 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.652467425452865e-06, |
| "logits/chosen": -0.43478575348854065, |
| "logits/rejected": -0.07904218137264252, |
| "logps/chosen": -461.2347717285156, |
| "logps/rejected": -556.1910400390625, |
| "loss": 0.4535, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.1336779594421387, |
| "rewards/margins": 1.1299854516983032, |
| "rewards/rejected": -3.2636635303497314, |
| "step": 4960 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.6417304072599787e-06, |
| "logits/chosen": -0.6369231939315796, |
| "logits/rejected": 0.5573645234107971, |
| "logps/chosen": -503.250732421875, |
| "logps/rejected": -592.5829467773438, |
| "loss": 0.531, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.3848960399627686, |
| "rewards/margins": 0.9315534830093384, |
| "rewards/rejected": -3.3164494037628174, |
| "step": 4970 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.6310113054765947e-06, |
| "logits/chosen": -0.6183720231056213, |
| "logits/rejected": 0.6410430073738098, |
| "logps/chosen": -529.9524536132812, |
| "logps/rejected": -604.1205444335938, |
| "loss": 0.5296, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.5189731121063232, |
| "rewards/margins": 1.104549765586853, |
| "rewards/rejected": -3.623522996902466, |
| "step": 4980 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.6203103438643591e-06, |
| "logits/chosen": -0.5253661870956421, |
| "logits/rejected": 0.08035139739513397, |
| "logps/chosen": -489.2454528808594, |
| "logps/rejected": -587.1968994140625, |
| "loss": 0.4981, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.3722403049468994, |
| "rewards/margins": 0.9075069427490234, |
| "rewards/rejected": -3.279747486114502, |
| "step": 4990 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 1.6096277458062417e-06, |
| "logits/chosen": 0.17417654395103455, |
| "logits/rejected": 0.0011325478553771973, |
| "logps/chosen": -417.90740966796875, |
| "logps/rejected": -536.4146118164062, |
| "loss": 0.4916, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -2.131730556488037, |
| "rewards/margins": 1.022687554359436, |
| "rewards/rejected": -3.1544182300567627, |
| "step": 5000 |
| }, |
| { |
| "epoch": 0.65, |
| "eval_logits/chosen": 1.3780100345611572, |
| "eval_logits/rejected": 2.116060256958008, |
| "eval_logps/chosen": -478.1451110839844, |
| "eval_logps/rejected": -558.6430053710938, |
| "eval_loss": 0.5087121725082397, |
| "eval_rewards/accuracies": 0.7300000190734863, |
| "eval_rewards/chosen": -2.096550941467285, |
| "eval_rewards/margins": 1.0040674209594727, |
| "eval_rewards/rejected": -3.100618362426758, |
| "eval_runtime": 1192.3166, |
| "eval_samples_per_second": 1.677, |
| "eval_steps_per_second": 0.839, |
| "step": 5000 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5989637343018705e-06, |
| "logits/chosen": -0.31024664640426636, |
| "logits/rejected": 0.315248966217041, |
| "logps/chosen": -463.248046875, |
| "logps/rejected": -594.2037963867188, |
| "loss": 0.4, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -1.942427635192871, |
| "rewards/margins": 1.2658753395080566, |
| "rewards/rejected": -3.2083029747009277, |
| "step": 5010 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5883185319628824e-06, |
| "logits/chosen": -0.7604548931121826, |
| "logits/rejected": 1.1575146913528442, |
| "logps/chosen": -514.0056762695312, |
| "logps/rejected": -576.3931884765625, |
| "loss": 0.4749, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.3210787773132324, |
| "rewards/margins": 1.0514533519744873, |
| "rewards/rejected": -3.372532367706299, |
| "step": 5020 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5776923610082695e-06, |
| "logits/chosen": -0.40637367963790894, |
| "logits/rejected": 0.5249187350273132, |
| "logps/chosen": -471.66650390625, |
| "logps/rejected": -582.8086547851562, |
| "loss": 0.4789, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.1091582775115967, |
| "rewards/margins": 1.3221811056137085, |
| "rewards/rejected": -3.431339979171753, |
| "step": 5030 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5670854432597433e-06, |
| "logits/chosen": -0.29735487699508667, |
| "logits/rejected": 0.3211146891117096, |
| "logps/chosen": -496.14306640625, |
| "logps/rejected": -521.6513061523438, |
| "loss": 0.5256, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.9637601375579834, |
| "rewards/margins": 0.7890554666519165, |
| "rewards/rejected": -2.7528157234191895, |
| "step": 5040 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.556498000137104e-06, |
| "logits/chosen": -0.15755796432495117, |
| "logits/rejected": 0.6389847993850708, |
| "logps/chosen": -417.54052734375, |
| "logps/rejected": -540.6213989257812, |
| "loss": 0.3919, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.8478233814239502, |
| "rewards/margins": 1.3877325057983398, |
| "rewards/rejected": -3.235556125640869, |
| "step": 5050 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5459302526536188e-06, |
| "logits/chosen": -0.493274986743927, |
| "logits/rejected": -0.03875049203634262, |
| "logps/chosen": -449.4249572753906, |
| "logps/rejected": -511.86492919921875, |
| "loss": 0.5453, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.8050098419189453, |
| "rewards/margins": 0.8924520611763, |
| "rewards/rejected": -2.697462320327759, |
| "step": 5060 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5353824214114075e-06, |
| "logits/chosen": -0.6162487268447876, |
| "logits/rejected": 0.14038828015327454, |
| "logps/chosen": -444.60736083984375, |
| "logps/rejected": -520.192138671875, |
| "loss": 0.5187, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.8896453380584717, |
| "rewards/margins": 0.8513407707214355, |
| "rewards/rejected": -2.7409861087799072, |
| "step": 5070 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 1.5248547265968373e-06, |
| "logits/chosen": -0.5640738606452942, |
| "logits/rejected": -0.3062174618244171, |
| "logps/chosen": -428.0648498535156, |
| "logps/rejected": -505.9144592285156, |
| "loss": 0.5312, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.8717830181121826, |
| "rewards/margins": 0.8837350606918335, |
| "rewards/rejected": -2.7555181980133057, |
| "step": 5080 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.5143473879759265e-06, |
| "logits/chosen": -0.9101265668869019, |
| "logits/rejected": 0.7031489610671997, |
| "logps/chosen": -447.75201416015625, |
| "logps/rejected": -512.3680419921875, |
| "loss": 0.5284, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.0770153999328613, |
| "rewards/margins": 1.0809861421585083, |
| "rewards/rejected": -3.158001184463501, |
| "step": 5090 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.5038606248897586e-06, |
| "logits/chosen": -0.36409762501716614, |
| "logits/rejected": 0.16153445839881897, |
| "logps/chosen": -540.3499755859375, |
| "logps/rejected": -589.29150390625, |
| "loss": 0.5806, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.5957887172698975, |
| "rewards/margins": 0.6685230135917664, |
| "rewards/rejected": -3.2643120288848877, |
| "step": 5100 |
| }, |
| { |
| "epoch": 0.67, |
| "eval_logits/chosen": 1.3594954013824463, |
| "eval_logits/rejected": 2.0896875858306885, |
| "eval_logps/chosen": -491.28375244140625, |
| "eval_logps/rejected": -572.3603515625, |
| "eval_loss": 0.5089035034179688, |
| "eval_rewards/accuracies": 0.7304999828338623, |
| "eval_rewards/chosen": -2.2279369831085205, |
| "eval_rewards/margins": 1.0098555088043213, |
| "eval_rewards/rejected": -3.237792491912842, |
| "eval_runtime": 1197.34, |
| "eval_samples_per_second": 1.67, |
| "eval_steps_per_second": 0.835, |
| "step": 5100 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.4933946562499008e-06, |
| "logits/chosen": -0.3227179944515228, |
| "logits/rejected": 0.6649230718612671, |
| "logps/chosen": -481.3219299316406, |
| "logps/rejected": -565.8980712890625, |
| "loss": 0.5032, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.204648733139038, |
| "rewards/margins": 1.1594634056091309, |
| "rewards/rejected": -3.364112377166748, |
| "step": 5110 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.482949700533835e-06, |
| "logits/chosen": -0.20165733993053436, |
| "logits/rejected": 0.22539961338043213, |
| "logps/chosen": -431.33514404296875, |
| "logps/rejected": -507.3214416503906, |
| "loss": 0.5052, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.136174440383911, |
| "rewards/margins": 0.8829553723335266, |
| "rewards/rejected": -3.019129753112793, |
| "step": 5120 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.4725259757803983e-06, |
| "logits/chosen": -0.3899513781070709, |
| "logits/rejected": 0.26980915665626526, |
| "logps/chosen": -543.8787841796875, |
| "logps/rejected": -611.9580688476562, |
| "loss": 0.5236, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.3227076530456543, |
| "rewards/margins": 1.0733088254928589, |
| "rewards/rejected": -3.3960158824920654, |
| "step": 5130 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.4621236995852314e-06, |
| "logits/chosen": -0.9624841809272766, |
| "logits/rejected": 0.41740506887435913, |
| "logps/chosen": -478.22625732421875, |
| "logps/rejected": -570.9786987304688, |
| "loss": 0.4712, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.1282503604888916, |
| "rewards/margins": 1.1468855142593384, |
| "rewards/rejected": -3.2751357555389404, |
| "step": 5140 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 1.4517430890962337e-06, |
| "logits/chosen": -1.003804087638855, |
| "logits/rejected": 0.8981930613517761, |
| "logps/chosen": -504.6312561035156, |
| "logps/rejected": -505.14471435546875, |
| "loss": 0.5248, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.165656328201294, |
| "rewards/margins": 0.9904571771621704, |
| "rewards/rejected": -3.1561131477355957, |
| "step": 5150 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.4413843610090342e-06, |
| "logits/chosen": -1.1295411586761475, |
| "logits/rejected": 0.7629106044769287, |
| "logps/chosen": -517.0686645507812, |
| "logps/rejected": -583.6041870117188, |
| "loss": 0.5029, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.2716588973999023, |
| "rewards/margins": 0.9855988621711731, |
| "rewards/rejected": -3.2572574615478516, |
| "step": 5160 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.4310477315624637e-06, |
| "logits/chosen": -0.5515908598899841, |
| "logits/rejected": 0.39060476422309875, |
| "logps/chosen": -463.7108459472656, |
| "logps/rejected": -558.4481201171875, |
| "loss": 0.603, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.238138198852539, |
| "rewards/margins": 0.9213913679122925, |
| "rewards/rejected": -3.1595299243927, |
| "step": 5170 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.420733416534045e-06, |
| "logits/chosen": 0.11524833738803864, |
| "logits/rejected": 0.43751248717308044, |
| "logps/chosen": -463.92352294921875, |
| "logps/rejected": -566.92578125, |
| "loss": 0.5296, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.1641011238098145, |
| "rewards/margins": 0.9625070691108704, |
| "rewards/rejected": -3.126608371734619, |
| "step": 5180 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.410441631235487e-06, |
| "logits/chosen": -0.6287524700164795, |
| "logits/rejected": -0.010029402561485767, |
| "logps/chosen": -475.24285888671875, |
| "logps/rejected": -560.3567504882812, |
| "loss": 0.4824, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9610084295272827, |
| "rewards/margins": 0.988863468170166, |
| "rewards/rejected": -2.949871778488159, |
| "step": 5190 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.4001725905081868e-06, |
| "logits/chosen": -0.594923198223114, |
| "logits/rejected": 0.47926950454711914, |
| "logps/chosen": -443.66815185546875, |
| "logps/rejected": -482.48907470703125, |
| "loss": 0.5027, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.1409497261047363, |
| "rewards/margins": 0.8297210931777954, |
| "rewards/rejected": -2.9706709384918213, |
| "step": 5200 |
| }, |
| { |
| "epoch": 0.68, |
| "eval_logits/chosen": 1.071418285369873, |
| "eval_logits/rejected": 1.8013691902160645, |
| "eval_logps/chosen": -458.1094970703125, |
| "eval_logps/rejected": -531.8433837890625, |
| "eval_loss": 0.5037881135940552, |
| "eval_rewards/accuracies": 0.737500011920929, |
| "eval_rewards/chosen": -1.8961946964263916, |
| "eval_rewards/margins": 0.936427116394043, |
| "eval_rewards/rejected": -2.8326218128204346, |
| "eval_runtime": 1205.2329, |
| "eval_samples_per_second": 1.659, |
| "eval_steps_per_second": 0.83, |
| "step": 5200 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.3899265087187507e-06, |
| "logits/chosen": -0.2585764527320862, |
| "logits/rejected": 0.0724058523774147, |
| "logps/chosen": -413.6856994628906, |
| "logps/rejected": -479.63323974609375, |
| "loss": 0.494, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8029801845550537, |
| "rewards/margins": 0.9064146876335144, |
| "rewards/rejected": -2.709394931793213, |
| "step": 5210 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.3797035997545144e-06, |
| "logits/chosen": -0.7779779434204102, |
| "logits/rejected": -0.05586876720190048, |
| "logps/chosen": -463.6990661621094, |
| "logps/rejected": -529.9464721679688, |
| "loss": 0.4447, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.761859655380249, |
| "rewards/margins": 1.0219517946243286, |
| "rewards/rejected": -2.7838118076324463, |
| "step": 5220 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 1.3695040770190816e-06, |
| "logits/chosen": -0.9211093187332153, |
| "logits/rejected": 0.09622125327587128, |
| "logps/chosen": -420.81951904296875, |
| "logps/rejected": -507.4986267089844, |
| "loss": 0.5245, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7863895893096924, |
| "rewards/margins": 0.9270517230033875, |
| "rewards/rejected": -2.7134411334991455, |
| "step": 5230 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3593281534278651e-06, |
| "logits/chosen": -0.5812297463417053, |
| "logits/rejected": -0.16451668739318848, |
| "logps/chosen": -413.26971435546875, |
| "logps/rejected": -532.844482421875, |
| "loss": 0.4485, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -1.8284438848495483, |
| "rewards/margins": 1.1127674579620361, |
| "rewards/rejected": -2.941211223602295, |
| "step": 5240 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3491760414036478e-06, |
| "logits/chosen": -0.8652269244194031, |
| "logits/rejected": 0.4699738919734955, |
| "logps/chosen": -487.8330993652344, |
| "logps/rejected": -502.11395263671875, |
| "loss": 0.6038, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.9750216007232666, |
| "rewards/margins": 0.6716020703315735, |
| "rewards/rejected": -2.6466238498687744, |
| "step": 5250 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3390479528721444e-06, |
| "logits/chosen": -0.4403937757015228, |
| "logits/rejected": 0.20110268890857697, |
| "logps/chosen": -475.3594665527344, |
| "logps/rejected": -581.604248046875, |
| "loss": 0.4959, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.161839723587036, |
| "rewards/margins": 1.0040169954299927, |
| "rewards/rejected": -3.1658565998077393, |
| "step": 5260 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3289440992575756e-06, |
| "logits/chosen": -0.5245973467826843, |
| "logits/rejected": 0.14496035873889923, |
| "logps/chosen": -487.95831298828125, |
| "logps/rejected": -544.9544677734375, |
| "loss": 0.4847, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.9425252676010132, |
| "rewards/margins": 0.839037299156189, |
| "rewards/rejected": -2.781562566757202, |
| "step": 5270 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3188646914782616e-06, |
| "logits/chosen": -1.1343739032745361, |
| "logits/rejected": 0.7802155613899231, |
| "logps/chosen": -519.5667724609375, |
| "logps/rejected": -522.45166015625, |
| "loss": 0.4612, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.960554838180542, |
| "rewards/margins": 0.9846093058586121, |
| "rewards/rejected": -2.945164203643799, |
| "step": 5280 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3088099399422109e-06, |
| "logits/chosen": -0.3516455292701721, |
| "logits/rejected": -0.15082845091819763, |
| "logps/chosen": -515.0338745117188, |
| "logps/rejected": -570.6123657226562, |
| "loss": 0.5449, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.2784736156463623, |
| "rewards/margins": 0.777103066444397, |
| "rewards/rejected": -3.0555763244628906, |
| "step": 5290 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.2987800545427353e-06, |
| "logits/chosen": -0.7941345572471619, |
| "logits/rejected": 0.2526698708534241, |
| "logps/chosen": -451.58319091796875, |
| "logps/rejected": -526.3946533203125, |
| "loss": 0.4554, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7788721323013306, |
| "rewards/margins": 1.0764901638031006, |
| "rewards/rejected": -2.8553624153137207, |
| "step": 5300 |
| }, |
| { |
| "epoch": 0.69, |
| "eval_logits/chosen": 1.155483365058899, |
| "eval_logits/rejected": 1.890524983406067, |
| "eval_logps/chosen": -463.9870300292969, |
| "eval_logps/rejected": -540.6599731445312, |
| "eval_loss": 0.505189836025238, |
| "eval_rewards/accuracies": 0.7329999804496765, |
| "eval_rewards/chosen": -1.9549702405929565, |
| "eval_rewards/margins": 0.9658184051513672, |
| "eval_rewards/rejected": -2.920788526535034, |
| "eval_runtime": 1180.1738, |
| "eval_samples_per_second": 1.695, |
| "eval_steps_per_second": 0.847, |
| "step": 5300 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.288775244654062e-06, |
| "logits/chosen": -0.3898960053920746, |
| "logits/rejected": -0.007041037082672119, |
| "logps/chosen": -516.4864501953125, |
| "logps/rejected": -554.4014282226562, |
| "loss": 0.5911, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.0645229816436768, |
| "rewards/margins": 0.8199642896652222, |
| "rewards/rejected": -2.8844873905181885, |
| "step": 5310 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.2787957191269696e-06, |
| "logits/chosen": -0.3055559992790222, |
| "logits/rejected": 0.39844006299972534, |
| "logps/chosen": -468.44097900390625, |
| "logps/rejected": -557.8650512695312, |
| "loss": 0.5665, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.0604255199432373, |
| "rewards/margins": 0.7497578859329224, |
| "rewards/rejected": -2.81018328666687, |
| "step": 5320 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.2688416862844193e-06, |
| "logits/chosen": -0.3095241189002991, |
| "logits/rejected": 0.23874731361865997, |
| "logps/chosen": -409.28021240234375, |
| "logps/rejected": -558.3905029296875, |
| "loss": 0.4496, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7695989608764648, |
| "rewards/margins": 1.3319097757339478, |
| "rewards/rejected": -3.101508378982544, |
| "step": 5330 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.2589133539172193e-06, |
| "logits/chosen": -0.9595244526863098, |
| "logits/rejected": 0.1507810652256012, |
| "logps/chosen": -486.26629638671875, |
| "logps/rejected": -545.3350830078125, |
| "loss": 0.432, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.7377183437347412, |
| "rewards/margins": 1.0882227420806885, |
| "rewards/rejected": -2.8259410858154297, |
| "step": 5340 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.249010929279672e-06, |
| "logits/chosen": -0.8987747430801392, |
| "logits/rejected": -0.06229814141988754, |
| "logps/chosen": -492.7890625, |
| "logps/rejected": -579.2696533203125, |
| "loss": 0.5204, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.0684757232666016, |
| "rewards/margins": 0.983476996421814, |
| "rewards/rejected": -3.051952600479126, |
| "step": 5350 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.2391346190852603e-06, |
| "logits/chosen": -0.8224785923957825, |
| "logits/rejected": 0.3233809769153595, |
| "logps/chosen": -468.3160095214844, |
| "logps/rejected": -521.6168823242188, |
| "loss": 0.6492, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.1017160415649414, |
| "rewards/margins": 0.6927504539489746, |
| "rewards/rejected": -2.794466495513916, |
| "step": 5360 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.2292846295023222e-06, |
| "logits/chosen": -0.5089510679244995, |
| "logits/rejected": 0.2342415750026703, |
| "logps/chosen": -515.0882568359375, |
| "logps/rejected": -560.9594116210938, |
| "loss": 0.5775, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.1873621940612793, |
| "rewards/margins": 0.6423591375350952, |
| "rewards/rejected": -2.829720973968506, |
| "step": 5370 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 1.2194611661497576e-06, |
| "logits/chosen": -0.40925416350364685, |
| "logits/rejected": 0.3876747190952301, |
| "logps/chosen": -480.6631774902344, |
| "logps/rejected": -569.6943969726562, |
| "loss": 0.4608, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.132833242416382, |
| "rewards/margins": 1.046400547027588, |
| "rewards/rejected": -3.1792335510253906, |
| "step": 5380 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.2096644340927247e-06, |
| "logits/chosen": -0.4503072202205658, |
| "logits/rejected": -0.0540265329182148, |
| "logps/chosen": -478.57281494140625, |
| "logps/rejected": -541.1819458007812, |
| "loss": 0.5218, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.94533371925354, |
| "rewards/margins": 0.8256410360336304, |
| "rewards/rejected": -2.770974636077881, |
| "step": 5390 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.19989463783837e-06, |
| "logits/chosen": -1.0149070024490356, |
| "logits/rejected": 0.19897064566612244, |
| "logps/chosen": -478.80670166015625, |
| "logps/rejected": -576.4320068359375, |
| "loss": 0.4521, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.8361622095108032, |
| "rewards/margins": 1.1377449035644531, |
| "rewards/rejected": -2.973906993865967, |
| "step": 5400 |
| }, |
| { |
| "epoch": 0.71, |
| "eval_logits/chosen": 1.1075793504714966, |
| "eval_logits/rejected": 1.8436723947525024, |
| "eval_logps/chosen": -467.6123962402344, |
| "eval_logps/rejected": -543.2982177734375, |
| "eval_loss": 0.5038532018661499, |
| "eval_rewards/accuracies": 0.7369999885559082, |
| "eval_rewards/chosen": -1.9912235736846924, |
| "eval_rewards/margins": 0.9559467434883118, |
| "eval_rewards/rejected": -2.9471704959869385, |
| "eval_runtime": 1197.7441, |
| "eval_samples_per_second": 1.67, |
| "eval_steps_per_second": 0.835, |
| "step": 5400 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.1901519813315495e-06, |
| "logits/chosen": -0.39034947752952576, |
| "logits/rejected": 0.27100151777267456, |
| "logps/chosen": -449.6952209472656, |
| "logps/rejected": -528.5842895507812, |
| "loss": 0.4638, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.9907958507537842, |
| "rewards/margins": 1.0050820112228394, |
| "rewards/rejected": -2.995877742767334, |
| "step": 5410 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.1804366679505798e-06, |
| "logits/chosen": -0.8501359820365906, |
| "logits/rejected": 0.513168215751648, |
| "logps/chosen": -513.7134399414062, |
| "logps/rejected": -545.6766967773438, |
| "loss": 0.5751, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.186096668243408, |
| "rewards/margins": 0.8673110008239746, |
| "rewards/rejected": -3.053407907485962, |
| "step": 5420 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.1707489005029877e-06, |
| "logits/chosen": -0.03637596219778061, |
| "logits/rejected": 0.2914156913757324, |
| "logps/chosen": -467.412353515625, |
| "logps/rejected": -572.0277099609375, |
| "loss": 0.4408, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.0735740661621094, |
| "rewards/margins": 1.2480766773223877, |
| "rewards/rejected": -3.321650743484497, |
| "step": 5430 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.1610888812212749e-06, |
| "logits/chosen": -0.569449245929718, |
| "logits/rejected": 0.4190862774848938, |
| "logps/chosen": -465.05218505859375, |
| "logps/rejected": -561.7803955078125, |
| "loss": 0.4018, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.994209885597229, |
| "rewards/margins": 1.1598801612854004, |
| "rewards/rejected": -3.1540896892547607, |
| "step": 5440 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.1514568117587035e-06, |
| "logits/chosen": -0.14006878435611725, |
| "logits/rejected": -0.16314168274402618, |
| "logps/chosen": -490.553955078125, |
| "logps/rejected": -563.6068115234375, |
| "loss": 0.5097, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.2165277004241943, |
| "rewards/margins": 0.8935906291007996, |
| "rewards/rejected": -3.1101181507110596, |
| "step": 5450 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 1.1418528931850781e-06, |
| "logits/chosen": -0.2969356179237366, |
| "logits/rejected": 0.7354004979133606, |
| "logps/chosen": -481.17120361328125, |
| "logps/rejected": -559.3109130859375, |
| "loss": 0.4667, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.0389533042907715, |
| "rewards/margins": 1.261207103729248, |
| "rewards/rejected": -3.3001601696014404, |
| "step": 5460 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.1322773259825563e-06, |
| "logits/chosen": -0.7932473421096802, |
| "logits/rejected": 0.7015919089317322, |
| "logps/chosen": -505.8838806152344, |
| "logps/rejected": -538.8965454101562, |
| "loss": 0.4926, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -2.2662813663482666, |
| "rewards/margins": 1.0218729972839355, |
| "rewards/rejected": -3.288154125213623, |
| "step": 5470 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.1227303100414552e-06, |
| "logits/chosen": -0.3481768071651459, |
| "logits/rejected": 0.19559481739997864, |
| "logps/chosen": -453.82781982421875, |
| "logps/rejected": -582.0299682617188, |
| "loss": 0.4953, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.3037056922912598, |
| "rewards/margins": 1.0858337879180908, |
| "rewards/rejected": -3.3895392417907715, |
| "step": 5480 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.113212044656087e-06, |
| "logits/chosen": -0.41049233078956604, |
| "logits/rejected": 0.1941242218017578, |
| "logps/chosen": -451.33038330078125, |
| "logps/rejected": -546.5862426757812, |
| "loss": 0.542, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.117616653442383, |
| "rewards/margins": 0.8589200973510742, |
| "rewards/rejected": -2.976536750793457, |
| "step": 5490 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.1037227285205951e-06, |
| "logits/chosen": 0.08584056794643402, |
| "logits/rejected": -0.11161471903324127, |
| "logps/chosen": -493.79400634765625, |
| "logps/rejected": -588.3775634765625, |
| "loss": 0.5869, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.3220810890197754, |
| "rewards/margins": 0.9301006197929382, |
| "rewards/rejected": -3.2521815299987793, |
| "step": 5500 |
| }, |
| { |
| "epoch": 0.72, |
| "eval_logits/chosen": 1.1574229001998901, |
| "eval_logits/rejected": 1.886451244354248, |
| "eval_logps/chosen": -485.5281066894531, |
| "eval_logps/rejected": -564.9520874023438, |
| "eval_loss": 0.5054470896720886, |
| "eval_rewards/accuracies": 0.7360000014305115, |
| "eval_rewards/chosen": -2.1703803539276123, |
| "eval_rewards/margins": 0.9933285117149353, |
| "eval_rewards/rejected": -3.1637091636657715, |
| "eval_runtime": 1194.4996, |
| "eval_samples_per_second": 1.674, |
| "eval_steps_per_second": 0.837, |
| "step": 5500 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.0942625597248028e-06, |
| "logits/chosen": -0.8230582475662231, |
| "logits/rejected": 0.5852819681167603, |
| "logps/chosen": -475.0633239746094, |
| "logps/rejected": -559.2849731445312, |
| "loss": 0.5008, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.2042269706726074, |
| "rewards/margins": 1.2232129573822021, |
| "rewards/rejected": -3.4274401664733887, |
| "step": 5510 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.0848317357500854e-06, |
| "logits/chosen": -0.4844978451728821, |
| "logits/rejected": 0.25016266107559204, |
| "logps/chosen": -517.09814453125, |
| "logps/rejected": -525.1922607421875, |
| "loss": 0.5246, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.2888691425323486, |
| "rewards/margins": 0.7657750248908997, |
| "rewards/rejected": -3.0546441078186035, |
| "step": 5520 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.0754304534652404e-06, |
| "logits/chosen": -0.05556102842092514, |
| "logits/rejected": -0.6705011129379272, |
| "logps/chosen": -480.46502685546875, |
| "logps/rejected": -580.5036010742188, |
| "loss": 0.5881, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.2701926231384277, |
| "rewards/margins": 0.6363264322280884, |
| "rewards/rejected": -2.9065189361572266, |
| "step": 5530 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 1.0660589091223854e-06, |
| "logits/chosen": -0.6355140209197998, |
| "logits/rejected": -0.15657804906368256, |
| "logps/chosen": -415.057373046875, |
| "logps/rejected": -514.7366943359375, |
| "loss": 0.4432, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.001784563064575, |
| "rewards/margins": 1.1080721616744995, |
| "rewards/rejected": -3.1098568439483643, |
| "step": 5540 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0567172983528534e-06, |
| "logits/chosen": -0.4708939492702484, |
| "logits/rejected": 0.3893592357635498, |
| "logps/chosen": -413.32177734375, |
| "logps/rejected": -507.1968688964844, |
| "loss": 0.4412, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.0215418338775635, |
| "rewards/margins": 0.9789319038391113, |
| "rewards/rejected": -3.000473976135254, |
| "step": 5550 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0474058161631168e-06, |
| "logits/chosen": -0.2970888018608093, |
| "logits/rejected": -0.11767357587814331, |
| "logps/chosen": -529.619140625, |
| "logps/rejected": -596.6094970703125, |
| "loss": 0.5635, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -2.1467816829681396, |
| "rewards/margins": 0.8911802172660828, |
| "rewards/rejected": -3.037961721420288, |
| "step": 5560 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0381246569307077e-06, |
| "logits/chosen": -0.535216212272644, |
| "logits/rejected": 0.25071626901626587, |
| "logps/chosen": -521.4472045898438, |
| "logps/rejected": -551.7024536132812, |
| "loss": 0.6022, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.2295970916748047, |
| "rewards/margins": 0.6778484582901001, |
| "rewards/rejected": -2.9074459075927734, |
| "step": 5570 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0288740144001722e-06, |
| "logits/chosen": -1.0160603523254395, |
| "logits/rejected": 0.10044028609991074, |
| "logps/chosen": -462.79730224609375, |
| "logps/rejected": -512.1188354492188, |
| "loss": 0.57, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.9899721145629883, |
| "rewards/margins": 0.9382216334342957, |
| "rewards/rejected": -2.928194046020508, |
| "step": 5580 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0196540816790127e-06, |
| "logits/chosen": -0.79753178358078, |
| "logits/rejected": 0.11815383285284042, |
| "logps/chosen": -427.5205078125, |
| "logps/rejected": -464.82562255859375, |
| "loss": 0.5304, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.933569312095642, |
| "rewards/margins": 0.8004889488220215, |
| "rewards/rejected": -2.7340588569641113, |
| "step": 5590 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0104650512336679e-06, |
| "logits/chosen": -0.9227102994918823, |
| "logits/rejected": 0.0573890320956707, |
| "logps/chosen": -458.4912109375, |
| "logps/rejected": -493.012939453125, |
| "loss": 0.5924, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.8620121479034424, |
| "rewards/margins": 0.8347771763801575, |
| "rewards/rejected": -2.696789264678955, |
| "step": 5600 |
| }, |
| { |
| "epoch": 0.73, |
| "eval_logits/chosen": 0.8215131759643555, |
| "eval_logits/rejected": 1.5324963331222534, |
| "eval_logps/chosen": -450.29345703125, |
| "eval_logps/rejected": -527.0138549804688, |
| "eval_loss": 0.5064495801925659, |
| "eval_rewards/accuracies": 0.7319999933242798, |
| "eval_rewards/chosen": -1.8180345296859741, |
| "eval_rewards/margins": 0.9662933349609375, |
| "eval_rewards/rejected": -2.784327745437622, |
| "eval_runtime": 1192.2692, |
| "eval_samples_per_second": 1.677, |
| "eval_steps_per_second": 0.839, |
| "step": 5600 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 1.0013071148854861e-06, |
| "logits/chosen": -0.46502774953842163, |
| "logits/rejected": -0.10884647071361542, |
| "logps/chosen": -406.987060546875, |
| "logps/rejected": -543.9107055664062, |
| "loss": 0.4149, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.725001573562622, |
| "rewards/margins": 1.3317546844482422, |
| "rewards/rejected": -3.0567562580108643, |
| "step": 5610 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.921804638067292e-07, |
| "logits/chosen": -0.9193164110183716, |
| "logits/rejected": 0.3256033957004547, |
| "logps/chosen": -473.353759765625, |
| "logps/rejected": -545.8274536132812, |
| "loss": 0.4718, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.0329298973083496, |
| "rewards/margins": 1.113075852394104, |
| "rewards/rejected": -3.146005868911743, |
| "step": 5620 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.830852885165749e-07, |
| "logits/chosen": -0.1020597368478775, |
| "logits/rejected": -0.6451666355133057, |
| "logps/chosen": -406.77081298828125, |
| "logps/rejected": -512.661865234375, |
| "loss": 0.5619, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9078305959701538, |
| "rewards/margins": 0.6681450605392456, |
| "rewards/rejected": -2.5759758949279785, |
| "step": 5630 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.740217788771453e-07, |
| "logits/chosen": -0.8085994720458984, |
| "logits/rejected": 0.037676215171813965, |
| "logps/chosen": -455.20208740234375, |
| "logps/rejected": -509.16680908203125, |
| "loss": 0.4926, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.7756178379058838, |
| "rewards/margins": 0.8752728700637817, |
| "rewards/rejected": -2.650890588760376, |
| "step": 5640 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.649901240895374e-07, |
| "logits/chosen": -0.11386583000421524, |
| "logits/rejected": 0.1979323923587799, |
| "logps/chosen": -440.43438720703125, |
| "logps/rejected": -523.9313354492188, |
| "loss": 0.5482, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9702411890029907, |
| "rewards/margins": 0.900561511516571, |
| "rewards/rejected": -2.870802640914917, |
| "step": 5650 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.559905126898803e-07, |
| "logits/chosen": -1.057208776473999, |
| "logits/rejected": 0.41706523299217224, |
| "logps/chosen": -467.60906982421875, |
| "logps/rejected": -559.2456665039062, |
| "loss": 0.3784, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.945220947265625, |
| "rewards/margins": 1.2823781967163086, |
| "rewards/rejected": -3.2275986671447754, |
| "step": 5660 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.470231325453958e-07, |
| "logits/chosen": -0.6564953923225403, |
| "logits/rejected": 0.41940754652023315, |
| "logps/chosen": -467.0869140625, |
| "logps/rejected": -520.7767944335938, |
| "loss": 0.5324, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.068354368209839, |
| "rewards/margins": 0.8983513116836548, |
| "rewards/rejected": -2.966705799102783, |
| "step": 5670 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.380881708504741e-07, |
| "logits/chosen": -0.29646626114845276, |
| "logits/rejected": 0.6381570100784302, |
| "logps/chosen": -402.63690185546875, |
| "logps/rejected": -467.7684020996094, |
| "loss": 0.4885, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.7983324527740479, |
| "rewards/margins": 0.9997227787971497, |
| "rewards/rejected": -2.798055410385132, |
| "step": 5680 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 9.291858141227733e-07, |
| "logits/chosen": -0.41397857666015625, |
| "logits/rejected": -0.3768884539604187, |
| "logps/chosen": -433.42816162109375, |
| "logps/rejected": -586.0889892578125, |
| "loss": 0.4259, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.8530666828155518, |
| "rewards/margins": 1.3527576923370361, |
| "rewards/rejected": -3.205824375152588, |
| "step": 5690 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 9.203162481993175e-07, |
| "logits/chosen": -0.9961411356925964, |
| "logits/rejected": -0.5173764228820801, |
| "logps/chosen": -484.4798278808594, |
| "logps/rejected": -600.5260009765625, |
| "loss": 0.4275, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.856370210647583, |
| "rewards/margins": 1.2594194412231445, |
| "rewards/rejected": -3.1157894134521484, |
| "step": 5700 |
| }, |
| { |
| "epoch": 0.75, |
| "eval_logits/chosen": 0.995962917804718, |
| "eval_logits/rejected": 1.7229074239730835, |
| "eval_logps/chosen": -469.19317626953125, |
| "eval_logps/rejected": -549.8818969726562, |
| "eval_loss": 0.5055263042449951, |
| "eval_rewards/accuracies": 0.734000027179718, |
| "eval_rewards/chosen": -2.0070316791534424, |
| "eval_rewards/margins": 1.0059758424758911, |
| "eval_rewards/rejected": -3.013007402420044, |
| "eval_runtime": 1227.9162, |
| "eval_samples_per_second": 1.629, |
| "eval_steps_per_second": 0.814, |
| "step": 5700 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 9.114796582326255e-07, |
| "logits/chosen": -1.0405280590057373, |
| "logits/rejected": 0.4770817756652832, |
| "logps/chosen": -472.912109375, |
| "logps/rejected": -529.1992797851562, |
| "loss": 0.5751, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.202134609222412, |
| "rewards/margins": 0.855462908744812, |
| "rewards/rejected": -3.0575973987579346, |
| "step": 5710 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 9.026762286868373e-07, |
| "logits/chosen": -0.797855794429779, |
| "logits/rejected": -0.14750805497169495, |
| "logps/chosen": -454.8798828125, |
| "logps/rejected": -599.2950439453125, |
| "loss": 0.418, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.9340975284576416, |
| "rewards/margins": 1.3620237112045288, |
| "rewards/rejected": -3.2961208820343018, |
| "step": 5720 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 8.939061433338722e-07, |
| "logits/chosen": -0.7406786680221558, |
| "logits/rejected": -0.19209416210651398, |
| "logps/chosen": -473.05560302734375, |
| "logps/rejected": -562.7134399414062, |
| "loss": 0.5286, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.0807507038116455, |
| "rewards/margins": 0.8903197050094604, |
| "rewards/rejected": -2.9710705280303955, |
| "step": 5730 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 8.851695852495867e-07, |
| "logits/chosen": -0.4927915632724762, |
| "logits/rejected": -0.5158900618553162, |
| "logps/chosen": -431.3685607910156, |
| "logps/rejected": -555.056884765625, |
| "loss": 0.5171, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.0437633991241455, |
| "rewards/margins": 1.1984370946884155, |
| "rewards/rejected": -3.2422003746032715, |
| "step": 5740 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 8.764667368099525e-07, |
| "logits/chosen": -0.2646760046482086, |
| "logits/rejected": 0.0725071132183075, |
| "logps/chosen": -458.35369873046875, |
| "logps/rejected": -541.1822509765625, |
| "loss": 0.5088, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.1217246055603027, |
| "rewards/margins": 1.0628745555877686, |
| "rewards/rejected": -3.184598922729492, |
| "step": 5750 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 8.677977796872541e-07, |
| "logits/chosen": -0.7068579792976379, |
| "logits/rejected": 0.5747413039207458, |
| "logps/chosen": -504.84881591796875, |
| "logps/rejected": -543.8521728515625, |
| "loss": 0.5338, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.211315870285034, |
| "rewards/margins": 1.0215401649475098, |
| "rewards/rejected": -3.232856273651123, |
| "step": 5760 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.591628948462913e-07, |
| "logits/chosen": -0.13534298539161682, |
| "logits/rejected": 0.21535761654376984, |
| "logps/chosen": -481.2478942871094, |
| "logps/rejected": -595.2032470703125, |
| "loss": 0.4979, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.09712553024292, |
| "rewards/margins": 1.1030064821243286, |
| "rewards/rejected": -3.200131893157959, |
| "step": 5770 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.505622625406054e-07, |
| "logits/chosen": -0.13635878264904022, |
| "logits/rejected": -0.033559300005435944, |
| "logps/chosen": -466.84637451171875, |
| "logps/rejected": -594.5919189453125, |
| "loss": 0.4774, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.1044180393218994, |
| "rewards/margins": 1.287061095237732, |
| "rewards/rejected": -3.391479015350342, |
| "step": 5780 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.419960623087129e-07, |
| "logits/chosen": 0.0363604798913002, |
| "logits/rejected": 0.037777043879032135, |
| "logps/chosen": -419.9930114746094, |
| "logps/rejected": -560.1504516601562, |
| "loss": 0.5144, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.1091957092285156, |
| "rewards/margins": 1.1116769313812256, |
| "rewards/rejected": -3.220872402191162, |
| "step": 5790 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.334644729703617e-07, |
| "logits/chosen": -0.32428237795829773, |
| "logits/rejected": -0.13345181941986084, |
| "logps/chosen": -442.7701721191406, |
| "logps/rejected": -567.7952270507812, |
| "loss": 0.4746, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.127009391784668, |
| "rewards/margins": 1.1997586488723755, |
| "rewards/rejected": -3.326767683029175, |
| "step": 5800 |
| }, |
| { |
| "epoch": 0.76, |
| "eval_logits/chosen": 1.1167664527893066, |
| "eval_logits/rejected": 1.8506929874420166, |
| "eval_logps/chosen": -489.1824951171875, |
| "eval_logps/rejected": -573.2806396484375, |
| "eval_loss": 0.5072213411331177, |
| "eval_rewards/accuracies": 0.7300000190734863, |
| "eval_rewards/chosen": -2.2069249153137207, |
| "eval_rewards/margins": 1.0400696992874146, |
| "eval_rewards/rejected": -3.2469944953918457, |
| "eval_runtime": 1256.1768, |
| "eval_samples_per_second": 1.592, |
| "eval_steps_per_second": 0.796, |
| "step": 5800 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.249676726227931e-07, |
| "logits/chosen": -0.2723647356033325, |
| "logits/rejected": 0.3790988326072693, |
| "logps/chosen": -546.9454345703125, |
| "logps/rejected": -591.0533447265625, |
| "loss": 0.5515, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.456743001937866, |
| "rewards/margins": 0.8524373173713684, |
| "rewards/rejected": -3.3091800212860107, |
| "step": 5810 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.165058386370314e-07, |
| "logits/chosen": -0.2161664515733719, |
| "logits/rejected": 0.19322898983955383, |
| "logps/chosen": -473.7935485839844, |
| "logps/rejected": -592.0934448242188, |
| "loss": 0.488, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.100954055786133, |
| "rewards/margins": 1.0307704210281372, |
| "rewards/rejected": -3.1317245960235596, |
| "step": 5820 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 8.080791476541721e-07, |
| "logits/chosen": -0.2463117390871048, |
| "logits/rejected": 0.10424462705850601, |
| "logps/chosen": -435.74786376953125, |
| "logps/rejected": -540.4898681640625, |
| "loss": 0.4526, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9524482488632202, |
| "rewards/margins": 1.2612793445587158, |
| "rewards/rejected": -3.2137274742126465, |
| "step": 5830 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 7.996877755817026e-07, |
| "logits/chosen": -0.7921704053878784, |
| "logits/rejected": 0.26169005036354065, |
| "logps/chosen": -465.552490234375, |
| "logps/rejected": -492.36956787109375, |
| "loss": 0.5762, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.133111000061035, |
| "rewards/margins": 0.5900529623031616, |
| "rewards/rejected": -2.7231638431549072, |
| "step": 5840 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.913318975898238e-07, |
| "logits/chosen": -0.8258234858512878, |
| "logits/rejected": 0.3698134422302246, |
| "logps/chosen": -555.0818481445312, |
| "logps/rejected": -597.4561767578125, |
| "loss": 0.5686, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.23042631149292, |
| "rewards/margins": 1.0009479522705078, |
| "rewards/rejected": -3.2313742637634277, |
| "step": 5850 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.830116881077992e-07, |
| "logits/chosen": -0.42663684487342834, |
| "logits/rejected": 0.7298521399497986, |
| "logps/chosen": -476.6568298339844, |
| "logps/rejected": -559.68310546875, |
| "loss": 0.4522, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.9448516368865967, |
| "rewards/margins": 1.070634126663208, |
| "rewards/rejected": -3.015486001968384, |
| "step": 5860 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.747273208203096e-07, |
| "logits/chosen": -0.327279657125473, |
| "logits/rejected": 0.10556366294622421, |
| "logps/chosen": -494.0747985839844, |
| "logps/rejected": -612.4318237304688, |
| "loss": 0.498, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.2703728675842285, |
| "rewards/margins": 1.0807685852050781, |
| "rewards/rejected": -3.3511414527893066, |
| "step": 5870 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.664789686638272e-07, |
| "logits/chosen": -0.753741443157196, |
| "logits/rejected": 0.26916906237602234, |
| "logps/chosen": -444.783447265625, |
| "logps/rejected": -573.6693115234375, |
| "loss": 0.4559, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.905149221420288, |
| "rewards/margins": 1.2002140283584595, |
| "rewards/rejected": -3.105363130569458, |
| "step": 5880 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.582668038230089e-07, |
| "logits/chosen": -0.6280697584152222, |
| "logits/rejected": 0.23410716652870178, |
| "logps/chosen": -483.71844482421875, |
| "logps/rejected": -548.9215087890625, |
| "loss": 0.6371, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -2.051710367202759, |
| "rewards/margins": 0.8779205083847046, |
| "rewards/rejected": -2.929630756378174, |
| "step": 5890 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.500909977271007e-07, |
| "logits/chosen": -0.55290687084198, |
| "logits/rejected": -0.04884564131498337, |
| "logps/chosen": -483.20465087890625, |
| "logps/rejected": -569.9694213867188, |
| "loss": 0.5033, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9960193634033203, |
| "rewards/margins": 1.1004588603973389, |
| "rewards/rejected": -3.096478223800659, |
| "step": 5900 |
| }, |
| { |
| "epoch": 0.77, |
| "eval_logits/chosen": 0.9675183892250061, |
| "eval_logits/rejected": 1.7070553302764893, |
| "eval_logps/chosen": -458.106201171875, |
| "eval_logps/rejected": -536.0161743164062, |
| "eval_loss": 0.5060694217681885, |
| "eval_rewards/accuracies": 0.7275000214576721, |
| "eval_rewards/chosen": -1.896161675453186, |
| "eval_rewards/margins": 0.9781885147094727, |
| "eval_rewards/rejected": -2.874350070953369, |
| "eval_runtime": 1176.3994, |
| "eval_samples_per_second": 1.7, |
| "eval_steps_per_second": 0.85, |
| "step": 5900 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.41951721046357e-07, |
| "logits/chosen": -0.6494289040565491, |
| "logits/rejected": 0.32405179738998413, |
| "logps/chosen": -428.957275390625, |
| "logps/rejected": -514.9093017578125, |
| "loss": 0.4849, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.763302206993103, |
| "rewards/margins": 0.9204890131950378, |
| "rewards/rejected": -2.683791399002075, |
| "step": 5910 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 7.338491436884787e-07, |
| "logits/chosen": -0.2216249257326126, |
| "logits/rejected": -0.0639793649315834, |
| "logps/chosen": -431.1812438964844, |
| "logps/rejected": -554.4732666015625, |
| "loss": 0.4382, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.016435146331787, |
| "rewards/margins": 1.169579267501831, |
| "rewards/rejected": -3.1860146522521973, |
| "step": 5920 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 7.257834347950693e-07, |
| "logits/chosen": -0.5539714097976685, |
| "logits/rejected": 0.5835639238357544, |
| "logps/chosen": -446.4956970214844, |
| "logps/rejected": -492.2032165527344, |
| "loss": 0.5289, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.8876628875732422, |
| "rewards/margins": 0.7628009915351868, |
| "rewards/rejected": -2.650463819503784, |
| "step": 5930 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 7.177547627380987e-07, |
| "logits/chosen": -0.4449450373649597, |
| "logits/rejected": -0.001817166805267334, |
| "logps/chosen": -462.5804748535156, |
| "logps/rejected": -559.3094482421875, |
| "loss": 0.4137, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.7668288946151733, |
| "rewards/margins": 1.125380277633667, |
| "rewards/rejected": -2.89220929145813, |
| "step": 5940 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 7.097632951163949e-07, |
| "logits/chosen": -0.49515801668167114, |
| "logits/rejected": 0.028278637677431107, |
| "logps/chosen": -472.94647216796875, |
| "logps/rejected": -543.9949340820312, |
| "loss": 0.5109, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7570574283599854, |
| "rewards/margins": 1.0789839029312134, |
| "rewards/rejected": -2.8360414505004883, |
| "step": 5950 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 7.018091987521386e-07, |
| "logits/chosen": -0.8645265698432922, |
| "logits/rejected": 0.3468485176563263, |
| "logps/chosen": -471.080810546875, |
| "logps/rejected": -537.1405029296875, |
| "loss": 0.5746, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.022472381591797, |
| "rewards/margins": 0.8672344088554382, |
| "rewards/rejected": -2.889706611633301, |
| "step": 5960 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 6.93892639687386e-07, |
| "logits/chosen": -0.655937671661377, |
| "logits/rejected": -0.05244150012731552, |
| "logps/chosen": -465.4962463378906, |
| "logps/rejected": -507.5498046875, |
| "loss": 0.5212, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.7037566900253296, |
| "rewards/margins": 0.9643377065658569, |
| "rewards/rejected": -2.6680946350097656, |
| "step": 5970 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 6.860137831806018e-07, |
| "logits/chosen": -0.5556502938270569, |
| "logits/rejected": -0.3047857880592346, |
| "logps/chosen": -465.64697265625, |
| "logps/rejected": -526.3079223632812, |
| "loss": 0.5657, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8184341192245483, |
| "rewards/margins": 0.9852198362350464, |
| "rewards/rejected": -2.8036537170410156, |
| "step": 5980 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 6.781727937032054e-07, |
| "logits/chosen": -0.3701472878456116, |
| "logits/rejected": 0.17485050857067108, |
| "logps/chosen": -410.296630859375, |
| "logps/rejected": -538.2269897460938, |
| "loss": 0.3979, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.5407458543777466, |
| "rewards/margins": 1.3232797384262085, |
| "rewards/rejected": -2.864025354385376, |
| "step": 5990 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.703698349361437e-07, |
| "logits/chosen": -0.6748332977294922, |
| "logits/rejected": 0.417569637298584, |
| "logps/chosen": -427.6536560058594, |
| "logps/rejected": -490.7718200683594, |
| "loss": 0.4517, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7474530935287476, |
| "rewards/margins": 0.9970604777336121, |
| "rewards/rejected": -2.744513988494873, |
| "step": 6000 |
| }, |
| { |
| "epoch": 0.79, |
| "eval_logits/chosen": 0.8156449198722839, |
| "eval_logits/rejected": 1.5612982511520386, |
| "eval_logps/chosen": -441.7278747558594, |
| "eval_logps/rejected": -516.7131958007812, |
| "eval_loss": 0.5105239152908325, |
| "eval_rewards/accuracies": 0.7264999747276306, |
| "eval_rewards/chosen": -1.7323784828186035, |
| "eval_rewards/margins": 0.9489423632621765, |
| "eval_rewards/rejected": -2.6813206672668457, |
| "eval_runtime": 1173.428, |
| "eval_samples_per_second": 1.704, |
| "eval_steps_per_second": 0.852, |
| "step": 6000 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.626050697664682e-07, |
| "logits/chosen": -0.5815194249153137, |
| "logits/rejected": 0.18539538979530334, |
| "logps/chosen": -448.04583740234375, |
| "logps/rejected": -505.4095153808594, |
| "loss": 0.4312, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.7882267236709595, |
| "rewards/margins": 1.0168880224227905, |
| "rewards/rejected": -2.80511474609375, |
| "step": 6010 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.548786602839404e-07, |
| "logits/chosen": -0.5067921876907349, |
| "logits/rejected": 0.06481216102838516, |
| "logps/chosen": -400.51556396484375, |
| "logps/rejected": -506.417236328125, |
| "loss": 0.4459, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.6396009922027588, |
| "rewards/margins": 1.3462693691253662, |
| "rewards/rejected": -2.985870361328125, |
| "step": 6020 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.471907677776426e-07, |
| "logits/chosen": -0.8723716735839844, |
| "logits/rejected": 0.1587451696395874, |
| "logps/chosen": -461.92376708984375, |
| "logps/rejected": -495.30029296875, |
| "loss": 0.5527, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7867259979248047, |
| "rewards/margins": 0.7085933685302734, |
| "rewards/rejected": -2.495319366455078, |
| "step": 6030 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.39541552732617e-07, |
| "logits/chosen": -0.41061049699783325, |
| "logits/rejected": -0.26837730407714844, |
| "logps/chosen": -455.27044677734375, |
| "logps/rejected": -573.69091796875, |
| "loss": 0.5308, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.901277780532837, |
| "rewards/margins": 0.9423490762710571, |
| "rewards/rejected": -2.8436269760131836, |
| "step": 6040 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.319311748265086e-07, |
| "logits/chosen": -0.3957150876522064, |
| "logits/rejected": 0.8088628053665161, |
| "logps/chosen": -577.0421752929688, |
| "logps/rejected": -617.0074462890625, |
| "loss": 0.5536, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.2539684772491455, |
| "rewards/margins": 1.012986660003662, |
| "rewards/rejected": -3.2669551372528076, |
| "step": 6050 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.243597929262404e-07, |
| "logits/chosen": -0.17666508257389069, |
| "logits/rejected": 0.17739292979240417, |
| "logps/chosen": -379.2519836425781, |
| "logps/rejected": -559.8756103515625, |
| "loss": 0.492, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7480781078338623, |
| "rewards/margins": 1.3037656545639038, |
| "rewards/rejected": -3.0518438816070557, |
| "step": 6060 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 6.168275650846875e-07, |
| "logits/chosen": -0.7394101619720459, |
| "logits/rejected": -0.0025915740989148617, |
| "logps/chosen": -452.69830322265625, |
| "logps/rejected": -493.15216064453125, |
| "loss": 0.5109, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.5727591514587402, |
| "rewards/margins": 0.9079796671867371, |
| "rewards/rejected": -2.480738878250122, |
| "step": 6070 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 6.093346485373863e-07, |
| "logits/chosen": -0.5971829295158386, |
| "logits/rejected": 0.7547621130943298, |
| "logps/chosen": -476.29693603515625, |
| "logps/rejected": -533.7948608398438, |
| "loss": 0.5113, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.9149093627929688, |
| "rewards/margins": 0.8861688375473022, |
| "rewards/rejected": -2.8010783195495605, |
| "step": 6080 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 6.018811996992455e-07, |
| "logits/chosen": -0.6492999792098999, |
| "logits/rejected": 0.5133501291275024, |
| "logps/chosen": -446.62933349609375, |
| "logps/rejected": -528.8438110351562, |
| "loss": 0.3894, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.6136525869369507, |
| "rewards/margins": 1.323101282119751, |
| "rewards/rejected": -2.936753749847412, |
| "step": 6090 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 5.944673741612866e-07, |
| "logits/chosen": -0.5191225409507751, |
| "logits/rejected": -0.2703114449977875, |
| "logps/chosen": -488.6419372558594, |
| "logps/rejected": -571.1328125, |
| "loss": 0.5071, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.037747859954834, |
| "rewards/margins": 0.8893720507621765, |
| "rewards/rejected": -2.9271202087402344, |
| "step": 6100 |
| }, |
| { |
| "epoch": 0.8, |
| "eval_logits/chosen": 0.9369565844535828, |
| "eval_logits/rejected": 1.6894502639770508, |
| "eval_logps/chosen": -454.8271789550781, |
| "eval_logps/rejected": -534.7506103515625, |
| "eval_loss": 0.5116304159164429, |
| "eval_rewards/accuracies": 0.7275000214576721, |
| "eval_rewards/chosen": -1.8633716106414795, |
| "eval_rewards/margins": 0.9983232617378235, |
| "eval_rewards/rejected": -2.861694574356079, |
| "eval_runtime": 1179.4032, |
| "eval_samples_per_second": 1.696, |
| "eval_steps_per_second": 0.848, |
| "step": 6100 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 5.870933266873916e-07, |
| "logits/chosen": -0.28973495960235596, |
| "logits/rejected": 0.016529571264982224, |
| "logps/chosen": -409.6407775878906, |
| "logps/rejected": -498.581787109375, |
| "loss": 0.5488, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.7589409351348877, |
| "rewards/margins": 0.8432859182357788, |
| "rewards/rejected": -2.602226734161377, |
| "step": 6110 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 5.797592112110734e-07, |
| "logits/chosen": -0.19786319136619568, |
| "logits/rejected": 0.2796550989151001, |
| "logps/chosen": -398.9677734375, |
| "logps/rejected": -460.76220703125, |
| "loss": 0.5853, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.8883498907089233, |
| "rewards/margins": 0.7973678708076477, |
| "rewards/rejected": -2.685717821121216, |
| "step": 6120 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 5.724651808322645e-07, |
| "logits/chosen": -0.3768353760242462, |
| "logits/rejected": 0.02244797721505165, |
| "logps/chosen": -426.26446533203125, |
| "logps/rejected": -552.0750122070312, |
| "loss": 0.4562, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7751553058624268, |
| "rewards/margins": 1.0461262464523315, |
| "rewards/rejected": -2.821281671524048, |
| "step": 6130 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 5.652113878141194e-07, |
| "logits/chosen": -0.6332122087478638, |
| "logits/rejected": 0.4572317600250244, |
| "logps/chosen": -385.82476806640625, |
| "logps/rejected": -482.82281494140625, |
| "loss": 0.5261, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.835436463356018, |
| "rewards/margins": 0.9859844446182251, |
| "rewards/rejected": -2.821420669555664, |
| "step": 6140 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 5.579979835798361e-07, |
| "logits/chosen": -0.4648275375366211, |
| "logits/rejected": 0.25945791602134705, |
| "logps/chosen": -436.0567321777344, |
| "logps/rejected": -553.8167114257812, |
| "loss": 0.4439, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.9036706686019897, |
| "rewards/margins": 1.2614033222198486, |
| "rewards/rejected": -3.165074110031128, |
| "step": 6150 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.508251187094932e-07, |
| "logits/chosen": -0.7267307043075562, |
| "logits/rejected": 0.3853886127471924, |
| "logps/chosen": -506.763427734375, |
| "logps/rejected": -537.0343017578125, |
| "loss": 0.6427, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.1635117530822754, |
| "rewards/margins": 0.7928264737129211, |
| "rewards/rejected": -2.9563381671905518, |
| "step": 6160 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.436929429369122e-07, |
| "logits/chosen": -0.4642343521118164, |
| "logits/rejected": 0.12719163298606873, |
| "logps/chosen": -440.3096618652344, |
| "logps/rejected": -517.3118286132812, |
| "loss": 0.5682, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.9598861932754517, |
| "rewards/margins": 0.9272395968437195, |
| "rewards/rejected": -2.8871257305145264, |
| "step": 6170 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.366016051465245e-07, |
| "logits/chosen": -0.5060332417488098, |
| "logits/rejected": 0.35478395223617554, |
| "logps/chosen": -435.00872802734375, |
| "logps/rejected": -564.3742065429688, |
| "loss": 0.4017, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.828107476234436, |
| "rewards/margins": 1.3373243808746338, |
| "rewards/rejected": -3.1654322147369385, |
| "step": 6180 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.295512533702701e-07, |
| "logits/chosen": -0.12531735002994537, |
| "logits/rejected": 0.5097149014472961, |
| "logps/chosen": -410.29974365234375, |
| "logps/rejected": -509.9228515625, |
| "loss": 0.532, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.8042405843734741, |
| "rewards/margins": 1.0363777875900269, |
| "rewards/rejected": -2.840618371963501, |
| "step": 6190 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.225420347845023e-07, |
| "logits/chosen": -0.7511438131332397, |
| "logits/rejected": 0.019868087023496628, |
| "logps/chosen": -490.9335021972656, |
| "logps/rejected": -552.3634033203125, |
| "loss": 0.6455, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.0654990673065186, |
| "rewards/margins": 0.7708456516265869, |
| "rewards/rejected": -2.8363449573516846, |
| "step": 6200 |
| }, |
| { |
| "epoch": 0.81, |
| "eval_logits/chosen": 0.9542487263679504, |
| "eval_logits/rejected": 1.7120394706726074, |
| "eval_logps/chosen": -456.4508056640625, |
| "eval_logps/rejected": -536.0125732421875, |
| "eval_loss": 0.5110178589820862, |
| "eval_rewards/accuracies": 0.7250000238418579, |
| "eval_rewards/chosen": -1.8796085119247437, |
| "eval_rewards/margins": 0.994705319404602, |
| "eval_rewards/rejected": -2.8743135929107666, |
| "eval_runtime": 1204.1855, |
| "eval_samples_per_second": 1.661, |
| "eval_steps_per_second": 0.83, |
| "step": 6200 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.155740957069186e-07, |
| "logits/chosen": -0.6087840795516968, |
| "logits/rejected": 0.2238648384809494, |
| "logps/chosen": -457.75323486328125, |
| "logps/rejected": -550.1719970703125, |
| "loss": 0.4296, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8891584873199463, |
| "rewards/margins": 1.2166895866394043, |
| "rewards/rejected": -3.1058480739593506, |
| "step": 6210 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 5.08647581593506e-07, |
| "logits/chosen": -0.24337653815746307, |
| "logits/rejected": 0.19924645125865936, |
| "logps/chosen": -430.85003662109375, |
| "logps/rejected": -523.4949340820312, |
| "loss": 0.4476, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.682952880859375, |
| "rewards/margins": 1.0686895847320557, |
| "rewards/rejected": -2.7516424655914307, |
| "step": 6220 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 5.017626370355014e-07, |
| "logits/chosen": -0.8117885589599609, |
| "logits/rejected": 0.5791851878166199, |
| "logps/chosen": -433.12237548828125, |
| "logps/rejected": -513.7730712890625, |
| "loss": 0.4269, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.7210042476654053, |
| "rewards/margins": 1.2424871921539307, |
| "rewards/rejected": -2.963491678237915, |
| "step": 6230 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.949194057563783e-07, |
| "logits/chosen": -0.7460896372795105, |
| "logits/rejected": 0.34127911925315857, |
| "logps/chosen": -460.87713623046875, |
| "logps/rejected": -520.8004150390625, |
| "loss": 0.4998, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.9422998428344727, |
| "rewards/margins": 1.0103784799575806, |
| "rewards/rejected": -2.9526784420013428, |
| "step": 6240 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.881180306088418e-07, |
| "logits/chosen": -0.7913476228713989, |
| "logits/rejected": 0.48180437088012695, |
| "logps/chosen": -464.13397216796875, |
| "logps/rejected": -523.867919921875, |
| "loss": 0.4899, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.8360408544540405, |
| "rewards/margins": 1.0949831008911133, |
| "rewards/rejected": -2.9310240745544434, |
| "step": 6250 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.813586535718512e-07, |
| "logits/chosen": -0.7295901775360107, |
| "logits/rejected": 1.1600463390350342, |
| "logps/chosen": -491.99139404296875, |
| "logps/rejected": -539.6505126953125, |
| "loss": 0.472, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.8388845920562744, |
| "rewards/margins": 1.2725069522857666, |
| "rewards/rejected": -3.111391544342041, |
| "step": 6260 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.746414157476506e-07, |
| "logits/chosen": -1.1962846517562866, |
| "logits/rejected": 0.240956112742424, |
| "logps/chosen": -408.2303771972656, |
| "logps/rejected": -522.8681640625, |
| "loss": 0.4429, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.7451229095458984, |
| "rewards/margins": 1.4104244709014893, |
| "rewards/rejected": -3.1555473804473877, |
| "step": 6270 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.679664573588294e-07, |
| "logits/chosen": -0.38419827818870544, |
| "logits/rejected": 0.4673822522163391, |
| "logps/chosen": -407.20782470703125, |
| "logps/rejected": -497.8232421875, |
| "loss": 0.4743, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.7693662643432617, |
| "rewards/margins": 1.090094804763794, |
| "rewards/rejected": -2.8594608306884766, |
| "step": 6280 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.6133391774538903e-07, |
| "logits/chosen": -0.8859823942184448, |
| "logits/rejected": 0.3551492989063263, |
| "logps/chosen": -481.630615234375, |
| "logps/rejected": -541.5256958007812, |
| "loss": 0.5493, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8950306177139282, |
| "rewards/margins": 1.0568093061447144, |
| "rewards/rejected": -2.9518399238586426, |
| "step": 6290 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.5474393536184214e-07, |
| "logits/chosen": -0.8629859685897827, |
| "logits/rejected": 0.4089199900627136, |
| "logps/chosen": -455.08062744140625, |
| "logps/rejected": -525.6192626953125, |
| "loss": 0.4796, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9132658243179321, |
| "rewards/margins": 1.0315920114517212, |
| "rewards/rejected": -2.9448578357696533, |
| "step": 6300 |
| }, |
| { |
| "epoch": 0.82, |
| "eval_logits/chosen": 1.020259976387024, |
| "eval_logits/rejected": 1.7784451246261597, |
| "eval_logps/chosen": -460.98785400390625, |
| "eval_logps/rejected": -543.0519409179688, |
| "eval_loss": 0.5111602544784546, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -1.924978494644165, |
| "eval_rewards/margins": 1.019729495048523, |
| "eval_rewards/rejected": -2.9447081089019775, |
| "eval_runtime": 1194.9348, |
| "eval_samples_per_second": 1.674, |
| "eval_steps_per_second": 0.837, |
| "step": 6300 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.4819664777431243e-07, |
| "logits/chosen": -0.18434950709342957, |
| "logits/rejected": 0.1741354763507843, |
| "logps/chosen": -412.4391174316406, |
| "logps/rejected": -475.8851013183594, |
| "loss": 0.5578, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.9152721166610718, |
| "rewards/margins": 0.8140000104904175, |
| "rewards/rejected": -2.72927188873291, |
| "step": 6310 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.416921916576722e-07, |
| "logits/chosen": -0.6540490388870239, |
| "logits/rejected": 0.48029765486717224, |
| "logps/chosen": -492.7520446777344, |
| "logps/rejected": -564.4111328125, |
| "loss": 0.5657, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.991019606590271, |
| "rewards/margins": 0.8219622373580933, |
| "rewards/rejected": -2.8129818439483643, |
| "step": 6320 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.352307027926828e-07, |
| "logits/chosen": -0.6802606582641602, |
| "logits/rejected": 0.12222106754779816, |
| "logps/chosen": -441.7935485839844, |
| "logps/rejected": -548.2308349609375, |
| "loss": 0.4019, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -1.7764034271240234, |
| "rewards/margins": 1.371129035949707, |
| "rewards/rejected": -3.1475327014923096, |
| "step": 6330 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.288123160631624e-07, |
| "logits/chosen": 0.02034485712647438, |
| "logits/rejected": 0.16564354300498962, |
| "logps/chosen": -435.69488525390625, |
| "logps/rejected": -519.12060546875, |
| "loss": 0.5531, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9254554510116577, |
| "rewards/margins": 0.9142245054244995, |
| "rewards/rejected": -2.839679718017578, |
| "step": 6340 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.224371654531731e-07, |
| "logits/chosen": -0.42781931161880493, |
| "logits/rejected": 0.401696115732193, |
| "logps/chosen": -449.87701416015625, |
| "logps/rejected": -493.78302001953125, |
| "loss": 0.5935, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9719626903533936, |
| "rewards/margins": 0.7792240381240845, |
| "rewards/rejected": -2.7511868476867676, |
| "step": 6350 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.1610538404421837e-07, |
| "logits/chosen": -0.12257415056228638, |
| "logits/rejected": -0.3362657427787781, |
| "logps/chosen": -416.6104431152344, |
| "logps/rejected": -562.0303344726562, |
| "loss": 0.4317, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.695469617843628, |
| "rewards/margins": 1.2062984704971313, |
| "rewards/rejected": -2.901768207550049, |
| "step": 6360 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.098171040124699e-07, |
| "logits/chosen": -0.9566270112991333, |
| "logits/rejected": 0.6268806457519531, |
| "logps/chosen": -514.7496337890625, |
| "logps/rejected": -540.041748046875, |
| "loss": 0.6099, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.057263135910034, |
| "rewards/margins": 0.8319833874702454, |
| "rewards/rejected": -2.889246702194214, |
| "step": 6370 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.03572456626006e-07, |
| "logits/chosen": -0.10828417539596558, |
| "logits/rejected": -0.13383126258850098, |
| "logps/chosen": -456.984130859375, |
| "logps/rejected": -528.4546508789062, |
| "loss": 0.4909, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.877745270729065, |
| "rewards/margins": 0.9030615091323853, |
| "rewards/rejected": -2.7808070182800293, |
| "step": 6380 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.9737157224207265e-07, |
| "logits/chosen": -0.623379647731781, |
| "logits/rejected": 0.0637415200471878, |
| "logps/chosen": -429.89349365234375, |
| "logps/rejected": -523.8753051757812, |
| "loss": 0.5763, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.002406597137451, |
| "rewards/margins": 0.9444215893745422, |
| "rewards/rejected": -2.9468283653259277, |
| "step": 6390 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.912145803043596e-07, |
| "logits/chosen": -0.6023394465446472, |
| "logits/rejected": 0.09874238818883896, |
| "logps/chosen": -470.8539123535156, |
| "logps/rejected": -520.394287109375, |
| "loss": 0.5568, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9863427877426147, |
| "rewards/margins": 0.7905007600784302, |
| "rewards/rejected": -2.776843309402466, |
| "step": 6400 |
| }, |
| { |
| "epoch": 0.84, |
| "eval_logits/chosen": 1.115229845046997, |
| "eval_logits/rejected": 1.8763548135757446, |
| "eval_logps/chosen": -463.8809509277344, |
| "eval_logps/rejected": -545.5327758789062, |
| "eval_loss": 0.5085515379905701, |
| "eval_rewards/accuracies": 0.7275000214576721, |
| "eval_rewards/chosen": -1.953909158706665, |
| "eval_rewards/margins": 1.0156067609786987, |
| "eval_rewards/rejected": -2.969515800476074, |
| "eval_runtime": 1193.0934, |
| "eval_samples_per_second": 1.676, |
| "eval_steps_per_second": 0.838, |
| "step": 6400 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.851016093403023e-07, |
| "logits/chosen": -0.14243006706237793, |
| "logits/rejected": 0.18332967162132263, |
| "logps/chosen": -435.7933044433594, |
| "logps/rejected": -534.3563842773438, |
| "loss": 0.5032, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.0880608558654785, |
| "rewards/margins": 1.0540637969970703, |
| "rewards/rejected": -3.142124652862549, |
| "step": 6410 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.7903278695839456e-07, |
| "logits/chosen": -0.13955774903297424, |
| "logits/rejected": -0.20281191170215607, |
| "logps/chosen": -465.5072326660156, |
| "logps/rejected": -532.3043212890625, |
| "loss": 0.5635, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9887361526489258, |
| "rewards/margins": 0.9076420068740845, |
| "rewards/rejected": -2.8963780403137207, |
| "step": 6420 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.7300823984552983e-07, |
| "logits/chosen": -0.5505388975143433, |
| "logits/rejected": -0.24104097485542297, |
| "logps/chosen": -418.98516845703125, |
| "logps/rejected": -541.846923828125, |
| "loss": 0.4913, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.963343620300293, |
| "rewards/margins": 1.0439479351043701, |
| "rewards/rejected": -3.007291793823242, |
| "step": 6430 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.670280937643503e-07, |
| "logits/chosen": -0.5132700204849243, |
| "logits/rejected": 0.6271919012069702, |
| "logps/chosen": -444.8404846191406, |
| "logps/rejected": -527.2811279296875, |
| "loss": 0.4893, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.886971116065979, |
| "rewards/margins": 1.1429336071014404, |
| "rewards/rejected": -3.029904842376709, |
| "step": 6440 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 3.610924735506274e-07, |
| "logits/chosen": -0.8992152214050293, |
| "logits/rejected": 0.8895284533500671, |
| "logps/chosen": -502.09051513671875, |
| "logps/rejected": -532.7025146484375, |
| "loss": 0.5641, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.00895094871521, |
| "rewards/margins": 0.9643281102180481, |
| "rewards/rejected": -2.9732792377471924, |
| "step": 6450 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.5520150311065316e-07, |
| "logits/chosen": -0.26017525792121887, |
| "logits/rejected": 0.5120875239372253, |
| "logps/chosen": -459.91436767578125, |
| "logps/rejected": -549.1633911132812, |
| "loss": 0.4416, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.8477706909179688, |
| "rewards/margins": 1.1013177633285522, |
| "rewards/rejected": -2.9490883350372314, |
| "step": 6460 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.493553054186527e-07, |
| "logits/chosen": -0.6854721307754517, |
| "logits/rejected": 0.3193429112434387, |
| "logps/chosen": -469.05230712890625, |
| "logps/rejected": -552.3963623046875, |
| "loss": 0.5335, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.117194652557373, |
| "rewards/margins": 0.8773033022880554, |
| "rewards/rejected": -2.994497776031494, |
| "step": 6470 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.4355400251421977e-07, |
| "logits/chosen": -0.15283913910388947, |
| "logits/rejected": 0.5103310346603394, |
| "logps/chosen": -456.74993896484375, |
| "logps/rejected": -499.0576171875, |
| "loss": 0.6538, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.07171368598938, |
| "rewards/margins": 0.6756519079208374, |
| "rewards/rejected": -2.747365713119507, |
| "step": 6480 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.3779771549976637e-07, |
| "logits/chosen": -0.47592344880104065, |
| "logits/rejected": 0.43167513608932495, |
| "logps/chosen": -438.9248962402344, |
| "logps/rejected": -505.8505859375, |
| "loss": 0.5395, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.9432265758514404, |
| "rewards/margins": 0.8998059034347534, |
| "rewards/rejected": -2.8430323600769043, |
| "step": 6490 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.3208656453799783e-07, |
| "logits/chosen": -0.7136383056640625, |
| "logits/rejected": 0.3548789918422699, |
| "logps/chosen": -434.717529296875, |
| "logps/rejected": -514.3789672851562, |
| "loss": 0.4335, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.8354284763336182, |
| "rewards/margins": 1.087013840675354, |
| "rewards/rejected": -2.9224421977996826, |
| "step": 6500 |
| }, |
| { |
| "epoch": 0.85, |
| "eval_logits/chosen": 1.1822218894958496, |
| "eval_logits/rejected": 1.9424830675125122, |
| "eval_logps/chosen": -468.96807861328125, |
| "eval_logps/rejected": -550.4982299804688, |
| "eval_loss": 0.5067179203033447, |
| "eval_rewards/accuracies": 0.7294999957084656, |
| "eval_rewards/chosen": -2.0047807693481445, |
| "eval_rewards/margins": 1.014390230178833, |
| "eval_rewards/rejected": -3.0191712379455566, |
| "eval_runtime": 1181.0567, |
| "eval_samples_per_second": 1.693, |
| "eval_steps_per_second": 0.847, |
| "step": 6500 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.2642066884940064e-07, |
| "logits/chosen": -0.5502122640609741, |
| "logits/rejected": 0.032435137778520584, |
| "logps/chosen": -474.4972229003906, |
| "logps/rejected": -589.9862670898438, |
| "loss": 0.594, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.066648006439209, |
| "rewards/margins": 1.129963755607605, |
| "rewards/rejected": -3.1966123580932617, |
| "step": 6510 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.2080014670975825e-07, |
| "logits/chosen": -0.5147031545639038, |
| "logits/rejected": -0.046147845685482025, |
| "logps/chosen": -433.7167053222656, |
| "logps/rejected": -496.712646484375, |
| "loss": 0.5683, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8814866542816162, |
| "rewards/margins": 0.8404488563537598, |
| "rewards/rejected": -2.721935510635376, |
| "step": 6520 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 3.152251154476765e-07, |
| "logits/chosen": -0.5666553378105164, |
| "logits/rejected": 0.2234136164188385, |
| "logps/chosen": -429.125732421875, |
| "logps/rejected": -543.9380493164062, |
| "loss": 0.4476, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.934643030166626, |
| "rewards/margins": 1.1717562675476074, |
| "rewards/rejected": -3.1063990592956543, |
| "step": 6530 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 3.0969569144214147e-07, |
| "logits/chosen": -0.9073036313056946, |
| "logits/rejected": 0.5169941186904907, |
| "logps/chosen": -476.5718688964844, |
| "logps/rejected": -547.4609985351562, |
| "loss": 0.4676, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.0425972938537598, |
| "rewards/margins": 0.9983150362968445, |
| "rewards/rejected": -3.040912628173828, |
| "step": 6540 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 3.042119901200824e-07, |
| "logits/chosen": -0.1191103607416153, |
| "logits/rejected": 0.11132284253835678, |
| "logps/chosen": -444.5992736816406, |
| "logps/rejected": -555.7691650390625, |
| "loss": 0.5972, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -2.1316585540771484, |
| "rewards/margins": 0.7841309309005737, |
| "rewards/rejected": -2.915789842605591, |
| "step": 6550 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.9877412595396726e-07, |
| "logits/chosen": -0.8357526063919067, |
| "logits/rejected": -0.019801050424575806, |
| "logps/chosen": -504.7652893066406, |
| "logps/rejected": -582.4708251953125, |
| "loss": 0.4626, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9639135599136353, |
| "rewards/margins": 1.1901118755340576, |
| "rewards/rejected": -3.1540255546569824, |
| "step": 6560 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.933822124594124e-07, |
| "logits/chosen": -0.2868785560131073, |
| "logits/rejected": 0.4834575057029724, |
| "logps/chosen": -471.68603515625, |
| "logps/rejected": -518.7160034179688, |
| "loss": 0.5799, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -2.0963294506073, |
| "rewards/margins": 0.8177496194839478, |
| "rewards/rejected": -2.914079189300537, |
| "step": 6570 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.880363621928106e-07, |
| "logits/chosen": -0.4423336982727051, |
| "logits/rejected": 0.6005432605743408, |
| "logps/chosen": -483.3235778808594, |
| "logps/rejected": -539.4915771484375, |
| "loss": 0.5047, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.052248239517212, |
| "rewards/margins": 0.9304612874984741, |
| "rewards/rejected": -2.9827091693878174, |
| "step": 6580 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.82736686748985e-07, |
| "logits/chosen": -0.6261542439460754, |
| "logits/rejected": 0.38417965173721313, |
| "logps/chosen": -470.73077392578125, |
| "logps/rejected": -533.3613891601562, |
| "loss": 0.4692, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9056422710418701, |
| "rewards/margins": 1.2279062271118164, |
| "rewards/rejected": -3.1335487365722656, |
| "step": 6590 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.774832967588556e-07, |
| "logits/chosen": -0.7487810850143433, |
| "logits/rejected": 0.46375662088394165, |
| "logps/chosen": -487.33544921875, |
| "logps/rejected": -587.102783203125, |
| "loss": 0.5263, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.039247512817383, |
| "rewards/margins": 1.248397707939148, |
| "rewards/rejected": -3.2876453399658203, |
| "step": 6600 |
| }, |
| { |
| "epoch": 0.86, |
| "eval_logits/chosen": 1.180580735206604, |
| "eval_logits/rejected": 1.9389982223510742, |
| "eval_logps/chosen": -465.3099060058594, |
| "eval_logps/rejected": -546.2759399414062, |
| "eval_loss": 0.5066229104995728, |
| "eval_rewards/accuracies": 0.7310000061988831, |
| "eval_rewards/chosen": -1.9681991338729858, |
| "eval_rewards/margins": 1.0087487697601318, |
| "eval_rewards/rejected": -2.976947784423828, |
| "eval_runtime": 1275.913, |
| "eval_samples_per_second": 1.568, |
| "eval_steps_per_second": 0.784, |
| "step": 6600 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.7227630188713326e-07, |
| "logits/chosen": -0.889278769493103, |
| "logits/rejected": 0.699964702129364, |
| "logps/chosen": -502.218017578125, |
| "logps/rejected": -545.4527587890625, |
| "loss": 0.504, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.0107197761535645, |
| "rewards/margins": 1.0104124546051025, |
| "rewards/rejected": -3.021131992340088, |
| "step": 6610 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.671158108300284e-07, |
| "logits/chosen": -0.6139100790023804, |
| "logits/rejected": -0.24325446784496307, |
| "logps/chosen": -461.9278259277344, |
| "logps/rejected": -540.6260986328125, |
| "loss": 0.5506, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.021791934967041, |
| "rewards/margins": 0.7798693776130676, |
| "rewards/rejected": -2.801661252975464, |
| "step": 6620 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.6200193131298376e-07, |
| "logits/chosen": -0.6526001691818237, |
| "logits/rejected": -0.10106471925973892, |
| "logps/chosen": -480.90618896484375, |
| "logps/rejected": -579.097900390625, |
| "loss": 0.4102, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -1.9294904470443726, |
| "rewards/margins": 1.2845728397369385, |
| "rewards/rejected": -3.2140636444091797, |
| "step": 6630 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.569347700884217e-07, |
| "logits/chosen": -0.69781893491745, |
| "logits/rejected": 0.686926543712616, |
| "logps/chosen": -466.5616149902344, |
| "logps/rejected": -538.9299926757812, |
| "loss": 0.473, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9398361444473267, |
| "rewards/margins": 1.1922200918197632, |
| "rewards/rejected": -3.13205623626709, |
| "step": 6640 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.5191443293352186e-07, |
| "logits/chosen": -0.3311184346675873, |
| "logits/rejected": 0.31631073355674744, |
| "logps/chosen": -480.02655029296875, |
| "logps/rejected": -576.1092529296875, |
| "loss": 0.5448, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.015752077102661, |
| "rewards/margins": 0.9452459216117859, |
| "rewards/rejected": -2.960998058319092, |
| "step": 6650 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.469410246480067e-07, |
| "logits/chosen": -0.21244564652442932, |
| "logits/rejected": 0.6931090950965881, |
| "logps/chosen": -439.51812744140625, |
| "logps/rejected": -541.0764770507812, |
| "loss": 0.4631, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.069284439086914, |
| "rewards/margins": 1.1991032361984253, |
| "rewards/rejected": -3.26838755607605, |
| "step": 6660 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.4201464905195955e-07, |
| "logits/chosen": -0.42047929763793945, |
| "logits/rejected": -0.40434974431991577, |
| "logps/chosen": -454.3592224121094, |
| "logps/rejected": -524.0152587890625, |
| "loss": 0.5885, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.9843008518218994, |
| "rewards/margins": 0.6817992925643921, |
| "rewards/rejected": -2.666100025177002, |
| "step": 6670 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.3713540898365196e-07, |
| "logits/chosen": -0.5671381950378418, |
| "logits/rejected": 0.32992106676101685, |
| "logps/chosen": -449.9239196777344, |
| "logps/rejected": -534.2097778320312, |
| "loss": 0.4748, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7972313165664673, |
| "rewards/margins": 1.1687263250350952, |
| "rewards/rejected": -2.9659574031829834, |
| "step": 6680 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.3230340629740166e-07, |
| "logits/chosen": -0.6218796968460083, |
| "logits/rejected": -0.1357102394104004, |
| "logps/chosen": -465.1273498535156, |
| "logps/rejected": -495.3822326660156, |
| "loss": 0.6258, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.9846065044403076, |
| "rewards/margins": 0.5649687647819519, |
| "rewards/rejected": -2.5495753288269043, |
| "step": 6690 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.2751874186144357e-07, |
| "logits/chosen": -1.0036194324493408, |
| "logits/rejected": 0.0544402189552784, |
| "logps/chosen": -472.434326171875, |
| "logps/rejected": -515.3161010742188, |
| "loss": 0.5263, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.8698005676269531, |
| "rewards/margins": 0.915500819683075, |
| "rewards/rejected": -2.7853012084960938, |
| "step": 6700 |
| }, |
| { |
| "epoch": 0.88, |
| "eval_logits/chosen": 1.1793560981750488, |
| "eval_logits/rejected": 1.9366413354873657, |
| "eval_logps/chosen": -465.6783752441406, |
| "eval_logps/rejected": -546.6119384765625, |
| "eval_loss": 0.5065844058990479, |
| "eval_rewards/accuracies": 0.7319999933242798, |
| "eval_rewards/chosen": -1.9718834161758423, |
| "eval_rewards/margins": 1.0084247589111328, |
| "eval_rewards/rejected": -2.9803082942962646, |
| "eval_runtime": 1228.3254, |
| "eval_samples_per_second": 1.628, |
| "eval_steps_per_second": 0.814, |
| "step": 6700 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.227815155558241e-07, |
| "logits/chosen": -0.715009331703186, |
| "logits/rejected": 0.11820618808269501, |
| "logps/chosen": -483.3104553222656, |
| "logps/rejected": -584.90478515625, |
| "loss": 0.5043, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.07963228225708, |
| "rewards/margins": 1.144766092300415, |
| "rewards/rejected": -3.224398136138916, |
| "step": 6710 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.1809182627031883e-07, |
| "logits/chosen": -0.9190937280654907, |
| "logits/rejected": 0.36419257521629333, |
| "logps/chosen": -471.60467529296875, |
| "logps/rejected": -543.3637084960938, |
| "loss": 0.5222, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8738210201263428, |
| "rewards/margins": 0.989733099937439, |
| "rewards/rejected": -2.8635544776916504, |
| "step": 6720 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.1344977190236372e-07, |
| "logits/chosen": 0.27573806047439575, |
| "logits/rejected": 0.18469476699829102, |
| "logps/chosen": -439.60107421875, |
| "logps/rejected": -560.4583740234375, |
| "loss": 0.4557, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9545615911483765, |
| "rewards/margins": 1.2339239120483398, |
| "rewards/rejected": -3.188485622406006, |
| "step": 6730 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.0885544935501656e-07, |
| "logits/chosen": -0.5300592184066772, |
| "logits/rejected": -0.35172906517982483, |
| "logps/chosen": -451.72027587890625, |
| "logps/rejected": -578.4183349609375, |
| "loss": 0.4219, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.9533073902130127, |
| "rewards/margins": 1.2389609813690186, |
| "rewards/rejected": -3.1922686100006104, |
| "step": 6740 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.0430895453492944e-07, |
| "logits/chosen": -0.5228718519210815, |
| "logits/rejected": -0.06577786058187485, |
| "logps/chosen": -495.9405212402344, |
| "logps/rejected": -537.6466674804688, |
| "loss": 0.5773, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.0149998664855957, |
| "rewards/margins": 0.7745741605758667, |
| "rewards/rejected": -2.789574384689331, |
| "step": 6750 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 1.9981038235035111e-07, |
| "logits/chosen": -0.12807337939739227, |
| "logits/rejected": -0.08245428651571274, |
| "logps/chosen": -441.65008544921875, |
| "logps/rejected": -548.439697265625, |
| "loss": 0.3802, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.779802680015564, |
| "rewards/margins": 1.3215081691741943, |
| "rewards/rejected": -3.101310968399048, |
| "step": 6760 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.9535982670914112e-07, |
| "logits/chosen": -0.55552738904953, |
| "logits/rejected": 0.4736878275871277, |
| "logps/chosen": -500.26171875, |
| "logps/rejected": -571.901611328125, |
| "loss": 0.5424, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.030512571334839, |
| "rewards/margins": 0.9621774554252625, |
| "rewards/rejected": -2.9926905632019043, |
| "step": 6770 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.9095738051681412e-07, |
| "logits/chosen": -0.19979307055473328, |
| "logits/rejected": 0.008223796263337135, |
| "logps/chosen": -459.77935791015625, |
| "logps/rejected": -547.7391357421875, |
| "loss": 0.5411, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.210814952850342, |
| "rewards/margins": 0.9136184453964233, |
| "rewards/rejected": -3.1244330406188965, |
| "step": 6780 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.8660313567459703e-07, |
| "logits/chosen": -0.09574131667613983, |
| "logits/rejected": -0.14765064418315887, |
| "logps/chosen": -426.57159423828125, |
| "logps/rejected": -542.1318359375, |
| "loss": 0.5432, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9452426433563232, |
| "rewards/margins": 1.1776217222213745, |
| "rewards/rejected": -3.1228644847869873, |
| "step": 6790 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.8229718307751165e-07, |
| "logits/chosen": -0.45521458983421326, |
| "logits/rejected": 0.7231898903846741, |
| "logps/chosen": -494.8271484375, |
| "logps/rejected": -564.8556518554688, |
| "loss": 0.4939, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -2.0975873470306396, |
| "rewards/margins": 1.2642689943313599, |
| "rewards/rejected": -3.361856460571289, |
| "step": 6800 |
| }, |
| { |
| "epoch": 0.89, |
| "eval_logits/chosen": 1.2238198518753052, |
| "eval_logits/rejected": 1.979459524154663, |
| "eval_logps/chosen": -470.53741455078125, |
| "eval_logps/rejected": -551.8628540039062, |
| "eval_loss": 0.5062793493270874, |
| "eval_rewards/accuracies": 0.7325000166893005, |
| "eval_rewards/chosen": -2.0204737186431885, |
| "eval_rewards/margins": 1.0123436450958252, |
| "eval_rewards/rejected": -3.0328176021575928, |
| "eval_runtime": 1180.1566, |
| "eval_samples_per_second": 1.695, |
| "eval_steps_per_second": 0.847, |
| "step": 6800 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.7803961261247864e-07, |
| "logits/chosen": -0.2857457995414734, |
| "logits/rejected": 0.201734259724617, |
| "logps/chosen": -467.9620666503906, |
| "logps/rejected": -601.2489013671875, |
| "loss": 0.4212, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.9273033142089844, |
| "rewards/margins": 1.3749698400497437, |
| "rewards/rejected": -3.3022735118865967, |
| "step": 6810 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.7383051315643772e-07, |
| "logits/chosen": -0.7285705804824829, |
| "logits/rejected": 0.5277493000030518, |
| "logps/chosen": -507.9571228027344, |
| "logps/rejected": -551.97607421875, |
| "loss": 0.58, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.2432968616485596, |
| "rewards/margins": 0.8464199304580688, |
| "rewards/rejected": -3.0897164344787598, |
| "step": 6820 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 1.6966997257449685e-07, |
| "logits/chosen": -0.5658080577850342, |
| "logits/rejected": 0.22322329878807068, |
| "logps/chosen": -476.03692626953125, |
| "logps/rejected": -543.3971557617188, |
| "loss": 0.526, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.0512733459472656, |
| "rewards/margins": 0.8895019292831421, |
| "rewards/rejected": -2.9407753944396973, |
| "step": 6830 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.6555807771809375e-07, |
| "logits/chosen": -0.6819769144058228, |
| "logits/rejected": 0.3543395698070526, |
| "logps/chosen": -454.736328125, |
| "logps/rejected": -513.91650390625, |
| "loss": 0.4701, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9653873443603516, |
| "rewards/margins": 1.1591241359710693, |
| "rewards/rejected": -3.124511241912842, |
| "step": 6840 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.6149491442318617e-07, |
| "logits/chosen": -0.37542515993118286, |
| "logits/rejected": -0.02834094688296318, |
| "logps/chosen": -455.55657958984375, |
| "logps/rejected": -529.7952880859375, |
| "loss": 0.5826, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.912413239479065, |
| "rewards/margins": 0.8809484243392944, |
| "rewards/rejected": -2.7933619022369385, |
| "step": 6850 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.5748056750845786e-07, |
| "logits/chosen": -0.7669013738632202, |
| "logits/rejected": 0.44335660338401794, |
| "logps/chosen": -499.58685302734375, |
| "logps/rejected": -512.37353515625, |
| "loss": 0.5071, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.2117278575897217, |
| "rewards/margins": 0.8127554655075073, |
| "rewards/rejected": -3.0244839191436768, |
| "step": 6860 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.5351512077355024e-07, |
| "logits/chosen": -0.5717117786407471, |
| "logits/rejected": 0.31644728779792786, |
| "logps/chosen": -493.24652099609375, |
| "logps/rejected": -646.7489624023438, |
| "loss": 0.4034, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9442113637924194, |
| "rewards/margins": 1.276631236076355, |
| "rewards/rejected": -3.2208428382873535, |
| "step": 6870 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.4959865699730902e-07, |
| "logits/chosen": -0.39092275500297546, |
| "logits/rejected": 0.6129297614097595, |
| "logps/chosen": -439.7764587402344, |
| "logps/rejected": -524.3287353515625, |
| "loss": 0.4971, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.0712218284606934, |
| "rewards/margins": 1.1431468725204468, |
| "rewards/rejected": -3.214369297027588, |
| "step": 6880 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.4573125793606202e-07, |
| "logits/chosen": -0.4366453289985657, |
| "logits/rejected": 0.452403724193573, |
| "logps/chosen": -431.4530334472656, |
| "logps/rejected": -524.9496459960938, |
| "loss": 0.5224, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.097776174545288, |
| "rewards/margins": 1.0594654083251953, |
| "rewards/rejected": -3.1572415828704834, |
| "step": 6890 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.4191300432190634e-07, |
| "logits/chosen": -0.45784568786621094, |
| "logits/rejected": 0.9001695513725281, |
| "logps/chosen": -504.549072265625, |
| "logps/rejected": -562.7247314453125, |
| "loss": 0.5763, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.3490102291107178, |
| "rewards/margins": 0.8059114217758179, |
| "rewards/rejected": -3.154921770095825, |
| "step": 6900 |
| }, |
| { |
| "epoch": 0.9, |
| "eval_logits/chosen": 1.202736258506775, |
| "eval_logits/rejected": 1.9579252004623413, |
| "eval_logps/chosen": -469.47125244140625, |
| "eval_logps/rejected": -550.4862670898438, |
| "eval_loss": 0.5060390830039978, |
| "eval_rewards/accuracies": 0.7329999804496765, |
| "eval_rewards/chosen": -2.009812116622925, |
| "eval_rewards/margins": 1.009238600730896, |
| "eval_rewards/rejected": -3.0190508365631104, |
| "eval_runtime": 1171.5664, |
| "eval_samples_per_second": 1.707, |
| "eval_steps_per_second": 0.854, |
| "step": 6900 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 1.381439758610284e-07, |
| "logits/chosen": -0.6321390271186829, |
| "logits/rejected": 0.20315834879875183, |
| "logps/chosen": -462.337158203125, |
| "logps/rejected": -521.3242797851562, |
| "loss": 0.5273, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.02864408493042, |
| "rewards/margins": 0.7589557766914368, |
| "rewards/rejected": -2.787600040435791, |
| "step": 6910 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.3442425123203596e-07, |
| "logits/chosen": -0.7073062658309937, |
| "logits/rejected": 0.02772808074951172, |
| "logps/chosen": -461.3926696777344, |
| "logps/rejected": -560.2494506835938, |
| "loss": 0.5085, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.046745538711548, |
| "rewards/margins": 1.014560580253601, |
| "rewards/rejected": -3.0613059997558594, |
| "step": 6920 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.3075390808431897e-07, |
| "logits/chosen": -0.37788501381874084, |
| "logits/rejected": 0.6019797325134277, |
| "logps/chosen": -435.75726318359375, |
| "logps/rejected": -511.6985778808594, |
| "loss": 0.4686, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9663892984390259, |
| "rewards/margins": 1.0337203741073608, |
| "rewards/rejected": -3.0001096725463867, |
| "step": 6930 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.271330230364262e-07, |
| "logits/chosen": -0.14386829733848572, |
| "logits/rejected": 0.21935054659843445, |
| "logps/chosen": -460.65582275390625, |
| "logps/rejected": -620.8492431640625, |
| "loss": 0.4858, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.145711898803711, |
| "rewards/margins": 1.1036556959152222, |
| "rewards/rejected": -3.2493674755096436, |
| "step": 6940 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.2356167167446698e-07, |
| "logits/chosen": -0.04140012338757515, |
| "logits/rejected": 0.4257062077522278, |
| "logps/chosen": -467.8836975097656, |
| "logps/rejected": -582.3963012695312, |
| "loss": 0.5375, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.3314528465270996, |
| "rewards/margins": 1.0109634399414062, |
| "rewards/rejected": -3.342416286468506, |
| "step": 6950 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.2003992855053326e-07, |
| "logits/chosen": -0.09045709669589996, |
| "logits/rejected": 0.434389591217041, |
| "logps/chosen": -432.42889404296875, |
| "logps/rejected": -556.8145751953125, |
| "loss": 0.5105, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.9629653692245483, |
| "rewards/margins": 1.2737910747528076, |
| "rewards/rejected": -3.2367560863494873, |
| "step": 6960 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.1656786718114239e-07, |
| "logits/chosen": -0.14372889697551727, |
| "logits/rejected": 0.17852702736854553, |
| "logps/chosen": -464.07659912109375, |
| "logps/rejected": -544.1351318359375, |
| "loss": 0.5259, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.1138415336608887, |
| "rewards/margins": 0.9006859064102173, |
| "rewards/rejected": -3.0145275592803955, |
| "step": 6970 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.1314556004570487e-07, |
| "logits/chosen": -0.35672527551651, |
| "logits/rejected": -0.19655278325080872, |
| "logps/chosen": -405.0440979003906, |
| "logps/rejected": -522.1256713867188, |
| "loss": 0.5576, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.9034448862075806, |
| "rewards/margins": 0.9194092750549316, |
| "rewards/rejected": -2.8228540420532227, |
| "step": 6980 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 1.0977307858500818e-07, |
| "logits/chosen": -0.7549588084220886, |
| "logits/rejected": 0.3601745367050171, |
| "logps/chosen": -438.78955078125, |
| "logps/rejected": -511.4344787597656, |
| "loss": 0.4611, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8593467473983765, |
| "rewards/margins": 1.0313808917999268, |
| "rewards/rejected": -2.8907275199890137, |
| "step": 6990 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 1.0645049319972789e-07, |
| "logits/chosen": -0.11657045036554337, |
| "logits/rejected": 0.3493362367153168, |
| "logps/chosen": -482.05938720703125, |
| "logps/rejected": -546.6885375976562, |
| "loss": 0.5062, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.1746017932891846, |
| "rewards/margins": 1.1042107343673706, |
| "rewards/rejected": -3.278812885284424, |
| "step": 7000 |
| }, |
| { |
| "epoch": 0.92, |
| "eval_logits/chosen": 1.2018110752105713, |
| "eval_logits/rejected": 1.957441806793213, |
| "eval_logps/chosen": -468.7945556640625, |
| "eval_logps/rejected": -549.6513671875, |
| "eval_loss": 0.5059376955032349, |
| "eval_rewards/accuracies": 0.7319999933242798, |
| "eval_rewards/chosen": -2.0030453205108643, |
| "eval_rewards/margins": 1.0076566934585571, |
| "eval_rewards/rejected": -3.010701894760132, |
| "eval_runtime": 1176.4736, |
| "eval_samples_per_second": 1.7, |
| "eval_steps_per_second": 0.85, |
| "step": 7000 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 1.0317787324895634e-07, |
| "logits/chosen": -0.35694876313209534, |
| "logits/rejected": 0.610944926738739, |
| "logps/chosen": -489.6722717285156, |
| "logps/rejected": -578.5293579101562, |
| "loss": 0.3914, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -2.016587495803833, |
| "rewards/margins": 1.268135666847229, |
| "rewards/rejected": -3.2847228050231934, |
| "step": 7010 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 9.995528704875635e-08, |
| "logits/chosen": -0.07247890532016754, |
| "logits/rejected": -0.2989567816257477, |
| "logps/chosen": -438.46728515625, |
| "logps/rejected": -547.0955810546875, |
| "loss": 0.5225, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.047827959060669, |
| "rewards/margins": 0.896246075630188, |
| "rewards/rejected": -2.9440741539001465, |
| "step": 7020 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 9.678280187073452e-08, |
| "logits/chosen": -0.27913758158683777, |
| "logits/rejected": 0.5736603140830994, |
| "logps/chosen": -463.31243896484375, |
| "logps/rejected": -556.4307861328125, |
| "loss": 0.3972, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.7716821432113647, |
| "rewards/margins": 1.4218103885650635, |
| "rewards/rejected": -3.1934924125671387, |
| "step": 7030 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 9.366048394063549e-08, |
| "logits/chosen": -0.41250452399253845, |
| "logits/rejected": -0.0662340372800827, |
| "logps/chosen": -456.4169006347656, |
| "logps/rejected": -561.72021484375, |
| "loss": 0.5114, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.850750207901001, |
| "rewards/margins": 0.9853949546813965, |
| "rewards/rejected": -2.8361451625823975, |
| "step": 7040 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 9.058839843696237e-08, |
| "logits/chosen": -0.4489797055721283, |
| "logits/rejected": 0.25591760873794556, |
| "logps/chosen": -483.81707763671875, |
| "logps/rejected": -559.30908203125, |
| "loss": 0.4706, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.0449726581573486, |
| "rewards/margins": 1.0600306987762451, |
| "rewards/rejected": -3.1050033569335938, |
| "step": 7050 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 8.756660948961299e-08, |
| "logits/chosen": -0.4514777660369873, |
| "logits/rejected": -0.20608548820018768, |
| "logps/chosen": -442.100341796875, |
| "logps/rejected": -545.4368286132812, |
| "loss": 0.5293, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.016775608062744, |
| "rewards/margins": 0.8498009443283081, |
| "rewards/rejected": -2.866576671600342, |
| "step": 7060 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 8.459518017854412e-08, |
| "logits/chosen": -0.6188634634017944, |
| "logits/rejected": 0.048554904758930206, |
| "logps/chosen": -459.34503173828125, |
| "logps/rejected": -503.36541748046875, |
| "loss": 0.5722, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.9700435400009155, |
| "rewards/margins": 0.6844844818115234, |
| "rewards/rejected": -2.6545281410217285, |
| "step": 7070 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 8.167417253245213e-08, |
| "logits/chosen": -0.7391661405563354, |
| "logits/rejected": 0.666763424873352, |
| "logps/chosen": -460.7491760253906, |
| "logps/rejected": -533.9959716796875, |
| "loss": 0.5173, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.060835123062134, |
| "rewards/margins": 0.9698716998100281, |
| "rewards/rejected": -3.0307066440582275, |
| "step": 7080 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 7.880364752747948e-08, |
| "logits/chosen": -0.42400145530700684, |
| "logits/rejected": -0.08642569929361343, |
| "logps/chosen": -453.1158142089844, |
| "logps/rejected": -535.9979248046875, |
| "loss": 0.5474, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.1991636753082275, |
| "rewards/margins": 0.8412439227104187, |
| "rewards/rejected": -3.04040789604187, |
| "step": 7090 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 7.598366508594245e-08, |
| "logits/chosen": -0.19622935354709625, |
| "logits/rejected": -0.13577821850776672, |
| "logps/chosen": -504.1654357910156, |
| "logps/rejected": -597.3096313476562, |
| "loss": 0.4432, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -2.007800340652466, |
| "rewards/margins": 1.198746919631958, |
| "rewards/rejected": -3.2065467834472656, |
| "step": 7100 |
| }, |
| { |
| "epoch": 0.93, |
| "eval_logits/chosen": 1.2115496397018433, |
| "eval_logits/rejected": 1.9674791097640991, |
| "eval_logps/chosen": -469.8140869140625, |
| "eval_logps/rejected": -550.7593994140625, |
| "eval_loss": 0.5058996081352234, |
| "eval_rewards/accuracies": 0.7329999804496765, |
| "eval_rewards/chosen": -2.0132408142089844, |
| "eval_rewards/margins": 1.0085415840148926, |
| "eval_rewards/rejected": -3.021782398223877, |
| "eval_runtime": 1186.4135, |
| "eval_samples_per_second": 1.686, |
| "eval_steps_per_second": 0.843, |
| "step": 7100 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 7.32142840750788e-08, |
| "logits/chosen": -0.5224785804748535, |
| "logits/rejected": 0.3103681802749634, |
| "logps/chosen": -489.0027770996094, |
| "logps/rejected": -570.6793212890625, |
| "loss": 0.4273, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.8569695949554443, |
| "rewards/margins": 1.2805449962615967, |
| "rewards/rejected": -3.137514591217041, |
| "step": 7110 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 7.049556230581872e-08, |
| "logits/chosen": -0.19431808590888977, |
| "logits/rejected": 0.6167198419570923, |
| "logps/chosen": -450.33013916015625, |
| "logps/rejected": -534.5645751953125, |
| "loss": 0.5363, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.142000675201416, |
| "rewards/margins": 1.0167112350463867, |
| "rewards/rejected": -3.1587119102478027, |
| "step": 7120 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 6.782755653158085e-08, |
| "logits/chosen": -0.4115025997161865, |
| "logits/rejected": 0.14585857093334198, |
| "logps/chosen": -475.48480224609375, |
| "logps/rejected": -537.6339111328125, |
| "loss": 0.517, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.024827480316162, |
| "rewards/margins": 0.8768397569656372, |
| "rewards/rejected": -2.901667356491089, |
| "step": 7130 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 6.521032244708375e-08, |
| "logits/chosen": -0.2687646448612213, |
| "logits/rejected": 0.2106965035200119, |
| "logps/chosen": -456.90948486328125, |
| "logps/rejected": -554.6942138671875, |
| "loss": 0.5336, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.990630865097046, |
| "rewards/margins": 0.9876958727836609, |
| "rewards/rejected": -2.9783265590667725, |
| "step": 7140 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 6.264391468718628e-08, |
| "logits/chosen": -0.7374650239944458, |
| "logits/rejected": 0.049775220453739166, |
| "logps/chosen": -459.86181640625, |
| "logps/rejected": -549.9932250976562, |
| "loss": 0.4637, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.8901011943817139, |
| "rewards/margins": 1.0680241584777832, |
| "rewards/rejected": -2.958125591278076, |
| "step": 7150 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 6.012838682574462e-08, |
| "logits/chosen": -0.6180638074874878, |
| "logits/rejected": 0.6540043950080872, |
| "logps/chosen": -475.832763671875, |
| "logps/rejected": -497.9046325683594, |
| "loss": 0.5072, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.0510411262512207, |
| "rewards/margins": 0.8002778887748718, |
| "rewards/rejected": -2.851318836212158, |
| "step": 7160 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 5.766379137449624e-08, |
| "logits/chosen": -0.36336010694503784, |
| "logits/rejected": 0.0748773068189621, |
| "logps/chosen": -423.7010192871094, |
| "logps/rejected": -554.7673950195312, |
| "loss": 0.4622, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9527654647827148, |
| "rewards/margins": 1.0933210849761963, |
| "rewards/rejected": -3.0460867881774902, |
| "step": 7170 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 5.525017978196295e-08, |
| "logits/chosen": -0.5020288825035095, |
| "logits/rejected": 0.6828367114067078, |
| "logps/chosen": -488.1974182128906, |
| "logps/rejected": -566.2771606445312, |
| "loss": 0.4981, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -2.0206267833709717, |
| "rewards/margins": 1.190063714981079, |
| "rewards/rejected": -3.21069073677063, |
| "step": 7180 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 5.288760243237545e-08, |
| "logits/chosen": -0.7169966697692871, |
| "logits/rejected": 0.34568727016448975, |
| "logps/chosen": -526.5534057617188, |
| "logps/rejected": -571.4498291015625, |
| "loss": 0.5102, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.1718175411224365, |
| "rewards/margins": 0.957715630531311, |
| "rewards/rejected": -3.129533052444458, |
| "step": 7190 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 5.0576108644623536e-08, |
| "logits/chosen": -0.5569435954093933, |
| "logits/rejected": 0.4495469927787781, |
| "logps/chosen": -521.062255859375, |
| "logps/rejected": -563.0753173828125, |
| "loss": 0.5294, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.2122786045074463, |
| "rewards/margins": 1.0275580883026123, |
| "rewards/rejected": -3.2398364543914795, |
| "step": 7200 |
| }, |
| { |
| "epoch": 0.94, |
| "eval_logits/chosen": 1.2122623920440674, |
| "eval_logits/rejected": 1.967880129814148, |
| "eval_logps/chosen": -469.9013671875, |
| "eval_logps/rejected": -550.8819580078125, |
| "eval_loss": 0.5059316158294678, |
| "eval_rewards/accuracies": 0.7315000295639038, |
| "eval_rewards/chosen": -2.014113426208496, |
| "eval_rewards/margins": 1.0088937282562256, |
| "eval_rewards/rejected": -3.0230071544647217, |
| "eval_runtime": 1186.2494, |
| "eval_samples_per_second": 1.686, |
| "eval_steps_per_second": 0.843, |
| "step": 7200 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 4.8315746671225296e-08, |
| "logits/chosen": -0.5705938339233398, |
| "logits/rejected": 0.5357488989830017, |
| "logps/chosen": -488.63177490234375, |
| "logps/rejected": -576.7957763671875, |
| "loss": 0.4553, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.8494971990585327, |
| "rewards/margins": 1.1614658832550049, |
| "rewards/rejected": -3.0109634399414062, |
| "step": 7210 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 4.6106563697320695e-08, |
| "logits/chosen": -0.6351941823959351, |
| "logits/rejected": 0.735435962677002, |
| "logps/chosen": -432.3277282714844, |
| "logps/rejected": -521.2852783203125, |
| "loss": 0.4985, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9638311862945557, |
| "rewards/margins": 1.1676876544952393, |
| "rewards/rejected": -3.131518840789795, |
| "step": 7220 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 4.394860583968624e-08, |
| "logits/chosen": 0.21465528011322021, |
| "logits/rejected": -0.21870703995227814, |
| "logps/chosen": -392.111328125, |
| "logps/rejected": -517.2110595703125, |
| "loss": 0.5165, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.8918077945709229, |
| "rewards/margins": 1.0091055631637573, |
| "rewards/rejected": -2.9009132385253906, |
| "step": 7230 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 4.1841918145771874e-08, |
| "logits/chosen": -0.24777980148792267, |
| "logits/rejected": 0.0036219656467437744, |
| "logps/chosen": -458.74249267578125, |
| "logps/rejected": -550.5509033203125, |
| "loss": 0.4325, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.8298200368881226, |
| "rewards/margins": 1.063430905342102, |
| "rewards/rejected": -2.8932509422302246, |
| "step": 7240 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 3.978654459276088e-08, |
| "logits/chosen": -0.7501617670059204, |
| "logits/rejected": 0.33591216802597046, |
| "logps/chosen": -512.9222412109375, |
| "logps/rejected": -573.7879028320312, |
| "loss": 0.4945, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.9614841938018799, |
| "rewards/margins": 1.2372934818267822, |
| "rewards/rejected": -3.198777437210083, |
| "step": 7250 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 3.778252808665284e-08, |
| "logits/chosen": -0.8246575593948364, |
| "logits/rejected": 0.21241407096385956, |
| "logps/chosen": -523.8837890625, |
| "logps/rejected": -519.209716796875, |
| "loss": 0.5693, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -2.0833640098571777, |
| "rewards/margins": 0.7381225824356079, |
| "rewards/rejected": -2.821486711502075, |
| "step": 7260 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 3.5829910461366023e-08, |
| "logits/chosen": 0.01369396410882473, |
| "logits/rejected": 0.08776617050170898, |
| "logps/chosen": -458.39581298828125, |
| "logps/rejected": -524.14013671875, |
| "loss": 0.6843, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -2.1614108085632324, |
| "rewards/margins": 0.7248591780662537, |
| "rewards/rejected": -2.8862698078155518, |
| "step": 7270 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 3.39287324778656e-08, |
| "logits/chosen": -0.7124800086021423, |
| "logits/rejected": 0.3487294912338257, |
| "logps/chosen": -543.2281494140625, |
| "logps/rejected": -586.5289916992188, |
| "loss": 0.6138, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.2240188121795654, |
| "rewards/margins": 0.852017879486084, |
| "rewards/rejected": -3.0760366916656494, |
| "step": 7280 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 3.207903382331262e-08, |
| "logits/chosen": -0.7476862072944641, |
| "logits/rejected": 0.375562846660614, |
| "logps/chosen": -478.54052734375, |
| "logps/rejected": -548.2950439453125, |
| "loss": 0.4735, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.7987592220306396, |
| "rewards/margins": 1.1323232650756836, |
| "rewards/rejected": -2.9310824871063232, |
| "step": 7290 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 3.028085311023443e-08, |
| "logits/chosen": -0.5295546054840088, |
| "logits/rejected": 0.3997403085231781, |
| "logps/chosen": -470.9478454589844, |
| "logps/rejected": -556.5192260742188, |
| "loss": 0.4488, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.9478727579116821, |
| "rewards/margins": 1.1733306646347046, |
| "rewards/rejected": -3.1212034225463867, |
| "step": 7300 |
| }, |
| { |
| "epoch": 0.96, |
| "eval_logits/chosen": 1.2130078077316284, |
| "eval_logits/rejected": 1.9687855243682861, |
| "eval_logps/chosen": -469.9289245605469, |
| "eval_logps/rejected": -550.9682006835938, |
| "eval_loss": 0.5058298707008362, |
| "eval_rewards/accuracies": 0.7319999933242798, |
| "eval_rewards/chosen": -2.0143890380859375, |
| "eval_rewards/margins": 1.00948166847229, |
| "eval_rewards/rejected": -3.0238709449768066, |
| "eval_runtime": 1188.4057, |
| "eval_samples_per_second": 1.683, |
| "eval_steps_per_second": 0.841, |
| "step": 7300 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.8534227875720576e-08, |
| "logits/chosen": -0.3639126420021057, |
| "logits/rejected": -0.1453981250524521, |
| "logps/chosen": -465.8466796875, |
| "logps/rejected": -564.0140991210938, |
| "loss": 0.5208, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.014688014984131, |
| "rewards/margins": 1.0618035793304443, |
| "rewards/rejected": -3.0764918327331543, |
| "step": 7310 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.683919458063705e-08, |
| "logits/chosen": -0.6341809630393982, |
| "logits/rejected": 0.7493041753768921, |
| "logps/chosen": -391.2576599121094, |
| "logps/rejected": -444.69024658203125, |
| "loss": 0.5043, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.8518688678741455, |
| "rewards/margins": 0.9767493009567261, |
| "rewards/rejected": -2.828618288040161, |
| "step": 7320 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.5195788608866345e-08, |
| "logits/chosen": -0.28447026014328003, |
| "logits/rejected": 0.6122376322746277, |
| "logps/chosen": -529.1468505859375, |
| "logps/rejected": -554.7265625, |
| "loss": 0.5484, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.015683889389038, |
| "rewards/margins": 0.9786075353622437, |
| "rewards/rejected": -2.994291305541992, |
| "step": 7330 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.3604044266569426e-08, |
| "logits/chosen": -0.8208459615707397, |
| "logits/rejected": 0.9798334836959839, |
| "logps/chosen": -484.0049743652344, |
| "logps/rejected": -558.9443969726562, |
| "loss": 0.4936, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.064974546432495, |
| "rewards/margins": 1.0787862539291382, |
| "rewards/rejected": -3.1437606811523438, |
| "step": 7340 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.2063994781468256e-08, |
| "logits/chosen": -0.2842608094215393, |
| "logits/rejected": 0.27055567502975464, |
| "logps/chosen": -452.8561096191406, |
| "logps/rejected": -518.8817749023438, |
| "loss": 0.5172, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8908157348632812, |
| "rewards/margins": 1.0217288732528687, |
| "rewards/rejected": -2.9125447273254395, |
| "step": 7350 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.057567230215246e-08, |
| "logits/chosen": -0.3838343322277069, |
| "logits/rejected": -0.392334908246994, |
| "logps/chosen": -468.7618103027344, |
| "logps/rejected": -558.9362182617188, |
| "loss": 0.5131, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.0254180431365967, |
| "rewards/margins": 0.84428870677948, |
| "rewards/rejected": -2.869706630706787, |
| "step": 7360 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 1.9139107897409303e-08, |
| "logits/chosen": -0.3951881229877472, |
| "logits/rejected": 0.7453802824020386, |
| "logps/chosen": -481.59222412109375, |
| "logps/rejected": -544.9617919921875, |
| "loss": 0.4279, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.9218416213989258, |
| "rewards/margins": 1.2567031383514404, |
| "rewards/rejected": -3.178544521331787, |
| "step": 7370 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.7754331555573656e-08, |
| "logits/chosen": -0.5559927821159363, |
| "logits/rejected": -0.07539238035678864, |
| "logps/chosen": -494.11053466796875, |
| "logps/rejected": -615.8850708007812, |
| "loss": 0.4835, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.090358018875122, |
| "rewards/margins": 0.9464312791824341, |
| "rewards/rejected": -3.0367894172668457, |
| "step": 7380 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.642137218390294e-08, |
| "logits/chosen": -0.41712522506713867, |
| "logits/rejected": 0.7344776391983032, |
| "logps/chosen": -474.60247802734375, |
| "logps/rejected": -519.6890869140625, |
| "loss": 0.5292, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.9687159061431885, |
| "rewards/margins": 0.937299370765686, |
| "rewards/rejected": -2.906015396118164, |
| "step": 7390 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.514025760797344e-08, |
| "logits/chosen": -0.8390815854072571, |
| "logits/rejected": 0.6691681742668152, |
| "logps/chosen": -529.1636962890625, |
| "logps/rejected": -570.968017578125, |
| "loss": 0.4747, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.024813175201416, |
| "rewards/margins": 1.0949513912200928, |
| "rewards/rejected": -3.119764804840088, |
| "step": 7400 |
| }, |
| { |
| "epoch": 0.97, |
| "eval_logits/chosen": 1.2122125625610352, |
| "eval_logits/rejected": 1.9678871631622314, |
| "eval_logps/chosen": -469.9052429199219, |
| "eval_logps/rejected": -550.9177856445312, |
| "eval_loss": 0.5057068467140198, |
| "eval_rewards/accuracies": 0.7325000166893005, |
| "eval_rewards/chosen": -2.0141522884368896, |
| "eval_rewards/margins": 1.0092144012451172, |
| "eval_rewards/rejected": -3.023366689682007, |
| "eval_runtime": 1179.1327, |
| "eval_samples_per_second": 1.696, |
| "eval_steps_per_second": 0.848, |
| "step": 7400 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.3911014571098835e-08, |
| "logits/chosen": -0.4136602282524109, |
| "logits/rejected": -0.011446630582213402, |
| "logps/chosen": -441.872314453125, |
| "logps/rejected": -560.9342041015625, |
| "loss": 0.4565, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9357706308364868, |
| "rewards/margins": 1.1018954515457153, |
| "rewards/rejected": -3.037665843963623, |
| "step": 7410 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.2733668733773685e-08, |
| "logits/chosen": -0.6516170501708984, |
| "logits/rejected": 0.346381813287735, |
| "logps/chosen": -461.1044006347656, |
| "logps/rejected": -534.1959228515625, |
| "loss": 0.4793, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.893280029296875, |
| "rewards/margins": 1.116193175315857, |
| "rewards/rejected": -3.0094730854034424, |
| "step": 7420 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.160824467313526e-08, |
| "logits/chosen": -0.6387825608253479, |
| "logits/rejected": 0.31129929423332214, |
| "logps/chosen": -525.2669067382812, |
| "logps/rejected": -617.6837158203125, |
| "loss": 0.4894, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.132877826690674, |
| "rewards/margins": 1.1318700313568115, |
| "rewards/rejected": -3.2647480964660645, |
| "step": 7430 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 1.0534765882453113e-08, |
| "logits/chosen": -0.9205204248428345, |
| "logits/rejected": 0.434356689453125, |
| "logps/chosen": -449.462890625, |
| "logps/rejected": -524.9171752929688, |
| "loss": 0.5114, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -1.9196815490722656, |
| "rewards/margins": 0.9198969602584839, |
| "rewards/rejected": -2.83957839012146, |
| "step": 7440 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 9.513254770636138e-09, |
| "logits/chosen": -0.3660666048526764, |
| "logits/rejected": 0.5724458694458008, |
| "logps/chosen": -530.5885009765625, |
| "logps/rejected": -592.3338623046875, |
| "loss": 0.5655, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -2.310126543045044, |
| "rewards/margins": 0.7468729019165039, |
| "rewards/rejected": -3.056999444961548, |
| "step": 7450 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 8.543732661767113e-09, |
| "logits/chosen": -0.14984729886054993, |
| "logits/rejected": 0.1948297917842865, |
| "logps/chosen": -473.29302978515625, |
| "logps/rejected": -562.1162109375, |
| "loss": 0.5471, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.9029672145843506, |
| "rewards/margins": 0.876223087310791, |
| "rewards/rejected": -2.7791905403137207, |
| "step": 7460 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 7.626219794655553e-09, |
| "logits/chosen": -0.7110680937767029, |
| "logits/rejected": 0.13748976588249207, |
| "logps/chosen": -441.283203125, |
| "logps/rejected": -585.7564086914062, |
| "loss": 0.4372, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.9341871738433838, |
| "rewards/margins": 1.3025288581848145, |
| "rewards/rejected": -3.236715793609619, |
| "step": 7470 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 6.7607353224163896e-09, |
| "logits/chosen": -0.5482068657875061, |
| "logits/rejected": 0.41012755036354065, |
| "logps/chosen": -485.97540283203125, |
| "logps/rejected": -520.5703125, |
| "loss": 0.5626, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -2.1319339275360107, |
| "rewards/margins": 0.7751830220222473, |
| "rewards/rejected": -2.907116413116455, |
| "step": 7480 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 5.947297312070554e-09, |
| "logits/chosen": -0.6903416514396667, |
| "logits/rejected": 0.9713506698608398, |
| "logps/chosen": -506.7986755371094, |
| "logps/rejected": -553.6888427734375, |
| "loss": 0.4687, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.060804843902588, |
| "rewards/margins": 1.1317999362945557, |
| "rewards/rejected": -3.1926045417785645, |
| "step": 7490 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 5.185922744166128e-09, |
| "logits/chosen": -0.4292621612548828, |
| "logits/rejected": 0.4434017539024353, |
| "logps/chosen": -476.03125, |
| "logps/rejected": -570.34375, |
| "loss": 0.4494, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9747473001480103, |
| "rewards/margins": 1.1069953441619873, |
| "rewards/rejected": -3.081742525100708, |
| "step": 7500 |
| }, |
| { |
| "epoch": 0.98, |
| "eval_logits/chosen": 1.212050437927246, |
| "eval_logits/rejected": 1.967947244644165, |
| "eval_logps/chosen": -469.93450927734375, |
| "eval_logps/rejected": -550.9584350585938, |
| "eval_loss": 0.5057631134986877, |
| "eval_rewards/accuracies": 0.7350000143051147, |
| "eval_rewards/chosen": -2.0144448280334473, |
| "eval_rewards/margins": 1.0093281269073486, |
| "eval_rewards/rejected": -3.023772954940796, |
| "eval_runtime": 1173.1475, |
| "eval_samples_per_second": 1.705, |
| "eval_steps_per_second": 0.852, |
| "step": 7500 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 4.476627512425558e-09, |
| "logits/chosen": -0.2599974274635315, |
| "logits/rejected": -0.058055657893419266, |
| "logps/chosen": -464.0921936035156, |
| "logps/rejected": -562.7515869140625, |
| "loss": 0.4728, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.9272934198379517, |
| "rewards/margins": 1.0648444890975952, |
| "rewards/rejected": -2.992137908935547, |
| "step": 7510 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 3.819426423412875e-09, |
| "logits/chosen": -0.6782014966011047, |
| "logits/rejected": 0.14074628055095673, |
| "logps/chosen": -494.22979736328125, |
| "logps/rejected": -555.0218505859375, |
| "loss": 0.6022, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.09846830368042, |
| "rewards/margins": 0.9207183122634888, |
| "rewards/rejected": -3.0191867351531982, |
| "step": 7520 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 3.2143331962256053e-09, |
| "logits/chosen": -0.24794983863830566, |
| "logits/rejected": 0.10213349014520645, |
| "logps/chosen": -464.79620361328125, |
| "logps/rejected": -570.9036865234375, |
| "loss": 0.4865, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -1.8833863735198975, |
| "rewards/margins": 1.067216157913208, |
| "rewards/rejected": -2.9506025314331055, |
| "step": 7530 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 2.6613604622066635e-09, |
| "logits/chosen": -0.35508936643600464, |
| "logits/rejected": -0.39137864112854004, |
| "logps/chosen": -444.24786376953125, |
| "logps/rejected": -548.8897705078125, |
| "loss": 0.5311, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.8555755615234375, |
| "rewards/margins": 0.892242431640625, |
| "rewards/rejected": -2.7478179931640625, |
| "step": 7540 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 2.1605197646826228e-09, |
| "logits/chosen": -0.4565364718437195, |
| "logits/rejected": 0.6792299151420593, |
| "logps/chosen": -427.64044189453125, |
| "logps/rejected": -537.2177734375, |
| "loss": 0.4213, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -1.849523901939392, |
| "rewards/margins": 1.3361475467681885, |
| "rewards/rejected": -3.185671806335449, |
| "step": 7550 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 1.711821558721405e-09, |
| "logits/chosen": -0.9418695569038391, |
| "logits/rejected": 0.49443039298057556, |
| "logps/chosen": -510.28033447265625, |
| "logps/rejected": -537.0841674804688, |
| "loss": 0.4911, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.0567002296447754, |
| "rewards/margins": 0.8696410059928894, |
| "rewards/rejected": -2.9263408184051514, |
| "step": 7560 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 1.3152752109149569e-09, |
| "logits/chosen": -0.4310362935066223, |
| "logits/rejected": 0.2431652992963791, |
| "logps/chosen": -481.66925048828125, |
| "logps/rejected": -550.9615478515625, |
| "loss": 0.5462, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.9845142364501953, |
| "rewards/margins": 0.914382815361023, |
| "rewards/rejected": -2.898897171020508, |
| "step": 7570 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 9.708889991830173e-10, |
| "logits/chosen": -0.8171736001968384, |
| "logits/rejected": 0.5985888242721558, |
| "logps/chosen": -484.5865783691406, |
| "logps/rejected": -540.4600219726562, |
| "loss": 0.4686, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -2.115016460418701, |
| "rewards/margins": 1.2058589458465576, |
| "rewards/rejected": -3.3208751678466797, |
| "step": 7580 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 6.786701125999218e-10, |
| "logits/chosen": 0.09515878558158875, |
| "logits/rejected": 0.33796870708465576, |
| "logps/chosen": -481.95538330078125, |
| "logps/rejected": -555.1029052734375, |
| "loss": 0.572, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -2.285205364227295, |
| "rewards/margins": 0.8154329061508179, |
| "rewards/rejected": -3.100637912750244, |
| "step": 7590 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 4.3862465124638873e-10, |
| "logits/chosen": -0.09601716697216034, |
| "logits/rejected": -0.01322057843208313, |
| "logps/chosen": -466.84674072265625, |
| "logps/rejected": -536.1382446289062, |
| "loss": 0.5319, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -2.0459887981414795, |
| "rewards/margins": 0.8657206296920776, |
| "rewards/rejected": -2.9117093086242676, |
| "step": 7600 |
| }, |
| { |
| "epoch": 0.99, |
| "eval_logits/chosen": 1.212050437927246, |
| "eval_logits/rejected": 1.967947244644165, |
| "eval_logps/chosen": -469.93450927734375, |
| "eval_logps/rejected": -550.9584350585938, |
| "eval_loss": 0.5057631134986877, |
| "eval_rewards/accuracies": 0.7350000143051147, |
| "eval_rewards/chosen": -2.0144448280334473, |
| "eval_rewards/margins": 1.0093281269073486, |
| "eval_rewards/rejected": -3.023772954940796, |
| "eval_runtime": 1169.363, |
| "eval_samples_per_second": 1.71, |
| "eval_steps_per_second": 0.855, |
| "step": 7600 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 2.507576260799005e-10, |
| "logits/chosen": -0.826134979724884, |
| "logits/rejected": 0.014235076494514942, |
| "logps/chosen": -475.70306396484375, |
| "logps/rejected": -575.2286376953125, |
| "loss": 0.4462, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.839888334274292, |
| "rewards/margins": 1.1129884719848633, |
| "rewards/rejected": -2.952876567840576, |
| "step": 7610 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 1.1507295883145253e-10, |
| "logits/chosen": -0.5861895680427551, |
| "logits/rejected": 0.36244645714759827, |
| "logps/chosen": -472.48651123046875, |
| "logps/rejected": -567.5523681640625, |
| "loss": 0.5132, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -1.945277214050293, |
| "rewards/margins": 0.905789852142334, |
| "rewards/rejected": -2.851067543029785, |
| "step": 7620 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 3.1573481923952156e-11, |
| "logits/chosen": -0.40739935636520386, |
| "logits/rejected": 0.2489284723997116, |
| "logps/chosen": -516.2659912109375, |
| "logps/rejected": -590.5760498046875, |
| "loss": 0.5475, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.035932779312134, |
| "rewards/margins": 1.092670202255249, |
| "rewards/rejected": -3.1286027431488037, |
| "step": 7630 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 2.609384119889313e-13, |
| "logits/chosen": -0.07999588549137115, |
| "logits/rejected": 0.025267338380217552, |
| "logps/chosen": -458.60418701171875, |
| "logps/rejected": -581.3901977539062, |
| "loss": 0.4471, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -2.0217761993408203, |
| "rewards/margins": 1.1485462188720703, |
| "rewards/rejected": -3.1703224182128906, |
| "step": 7640 |
| }, |
| { |
| "epoch": 1.0, |
| "step": 7641, |
| "total_flos": 0.0, |
| "train_loss": 0.0026284047499827543, |
| "train_runtime": 361.3652, |
| "train_samples_per_second": 169.178, |
| "train_steps_per_second": 21.145 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 7641, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
