Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,68 @@
+---
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+library_name: transformers
+model_name: Qwen2.5-1.5B-Open-R1-GRPO
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Open-R1-GRPO
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="coolcoolad/Qwen2.5-1.5B-Open-R1-GRPO", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.16.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.05497839203927236,
+    "train_runtime": 37424.3352,
+    "train_samples": 72441,
+    "train_samples_per_second": 1.936,
+    "train_steps_per_second": 0.01
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.49.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a42cec6a03e65c16a9cfc32d75c9173b35e1b91ee3ff3a0940c6c4c8c6c5a82f
+size 3087467144

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.05497839203927236,
+    "train_runtime": 37424.3352,
+    "train_samples": 72441,
+    "train_samples_per_second": 1.936,
+    "train_steps_per_second": 0.01
+}

trainer_state.json

@@ -0,0 +1,1065 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9991717740599636,
+  "eval_steps": 100,
+  "global_step": 377,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 390.31797828674314,
+      "epoch": 0.013251615040583071,
+      "grad_norm": 0.7148311400411296,
+      "kl": 0.00022041797637939453,
+      "learning_rate": 2.631578947368421e-06,
+      "loss": 0.0,
+      "reward": 0.6169271022081375,
+      "reward_std": 0.39014692828059194,
+      "rewards/accuracy_reward": 0.15598958749324082,
+      "rewards/format_reward": 0.4609375115483999,
+      "step": 5
+    },
+    {
+      "completion_length": 260.06250648498536,
+      "epoch": 0.026503230081166142,
+      "grad_norm": 0.8887729352532353,
+      "kl": 0.12361564636230468,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 0.0049,
+      "reward": 0.9091146066784859,
+      "reward_std": 0.2958403632976115,
+      "rewards/accuracy_reward": 0.09218750270083546,
+      "rewards/format_reward": 0.8169271022081375,
+      "step": 10
+    },
+    {
+      "completion_length": 217.22448501586913,
+      "epoch": 0.03975484512174921,
+      "grad_norm": 0.460053504280271,
+      "kl": 0.05746002197265625,
+      "learning_rate": 7.894736842105265e-06,
+      "loss": 0.0023,
+      "reward": 0.9986979469656945,
+      "reward_std": 0.22242207610979675,
+      "rewards/accuracy_reward": 0.0809895850950852,
+      "rewards/format_reward": 0.9177083529531955,
+      "step": 15
+    },
+    {
+      "completion_length": 121.37604475021362,
+      "epoch": 0.053006460162332285,
+      "grad_norm": 5.54386635888608,
+      "kl": 0.128759765625,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 0.0052,
+      "reward": 1.024479192495346,
+      "reward_std": 0.20674621588550507,
+      "rewards/accuracy_reward": 0.0895833360729739,
+      "rewards/format_reward": 0.9348958514630794,
+      "step": 20
+    },
+    {
+      "completion_length": 41.84296984672547,
+      "epoch": 0.06625807520291535,
+      "grad_norm": 0.632236355290783,
+      "kl": 0.42974853515625,
+      "learning_rate": 1.3157894736842108e-05,
+      "loss": 0.0172,
+      "reward": 1.0666666977107524,
+      "reward_std": 0.1505972647573799,
+      "rewards/accuracy_reward": 0.09973958600312471,
+      "rewards/format_reward": 0.966927096992731,
+      "step": 25
+    },
+    {
+      "completion_length": 28.13645920753479,
+      "epoch": 0.07950969024349842,
+      "grad_norm": 0.7530131466776888,
+      "kl": 0.616943359375,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 0.0247,
+      "reward": 1.1552083767950534,
+      "reward_std": 0.156400490924716,
+      "rewards/accuracy_reward": 0.16796875484287738,
+      "rewards/format_reward": 0.9872395932674408,
+      "step": 30
+    },
+    {
+      "completion_length": 59.088022685050966,
+      "epoch": 0.0927613052840815,
+      "grad_norm": 0.4466519788343623,
+      "kl": 0.7394287109375,
+      "learning_rate": 1.8421052631578947e-05,
+      "loss": 0.0296,
+      "reward": 1.128645869344473,
+      "reward_std": 0.166742659593001,
+      "rewards/accuracy_reward": 0.15781250512227415,
+      "rewards/format_reward": 0.9708333484828472,
+      "step": 35
+    },
+    {
+      "completion_length": 43.010157227516174,
+      "epoch": 0.10601292032466457,
+      "grad_norm": 0.3950304548883434,
+      "kl": 0.627587890625,
+      "learning_rate": 1.9998282416292057e-05,
+      "loss": 0.0251,
+      "reward": 1.1283854514360427,
+      "reward_std": 0.1949546963442117,
+      "rewards/accuracy_reward": 0.17317708781920374,
+      "rewards/format_reward": 0.9552083536982536,
+      "step": 40
+    },
+    {
+      "completion_length": 46.660938787460324,
+      "epoch": 0.11926453536524764,
+      "grad_norm": 0.32538591301168335,
+      "kl": 0.62978515625,
+      "learning_rate": 1.9978966374934255e-05,
+      "loss": 0.0252,
+      "reward": 1.164843785762787,
+      "reward_std": 0.1441725050099194,
+      "rewards/accuracy_reward": 0.18489583847112953,
+      "rewards/format_reward": 0.9799479305744171,
+      "step": 45
+    },
+    {
+      "completion_length": 105.41771154403686,
+      "epoch": 0.1325161504058307,
+      "grad_norm": 1.1943293073051344,
+      "kl": 0.5697021484375,
+      "learning_rate": 1.993822891578708e-05,
+      "loss": 0.0228,
+      "reward": 1.132812535762787,
+      "reward_std": 0.17938134195283056,
+      "rewards/accuracy_reward": 0.15989583814516664,
+      "rewards/format_reward": 0.9729166820645332,
+      "step": 50
+    },
+    {
+      "completion_length": 85.27838830947876,
+      "epoch": 0.1457677654464138,
+      "grad_norm": 37.675877105412894,
+      "kl": 0.98544921875,
+      "learning_rate": 1.9876157488201426e-05,
+      "loss": 0.0394,
+      "reward": 1.1236979596316814,
+      "reward_std": 0.16016151797957717,
+      "rewards/accuracy_reward": 0.14348958660848438,
+      "rewards/format_reward": 0.9802083469927311,
+      "step": 55
+    },
+    {
+      "completion_length": 94.97552375793457,
+      "epoch": 0.15901938048699685,
+      "grad_norm": 0.8524652073324807,
+      "kl": 0.685546875,
+      "learning_rate": 1.9792885338240375e-05,
+      "loss": 0.0274,
+      "reward": 1.0893229462206364,
+      "reward_std": 0.19034504797309637,
+      "rewards/accuracy_reward": 0.13645833721384407,
+      "rewards/format_reward": 0.9528646036982537,
+      "step": 60
+    },
+    {
+      "completion_length": 70.53073163032532,
+      "epoch": 0.17227099552757993,
+      "grad_norm": 0.3478501171295835,
+      "kl": 0.757666015625,
+      "learning_rate": 1.9688591222645607e-05,
+      "loss": 0.0303,
+      "reward": 1.1075521253049374,
+      "reward_std": 0.2081975589040667,
+      "rewards/accuracy_reward": 0.17031250472646206,
+      "rewards/format_reward": 0.9372396014630795,
+      "step": 65
+    },
+    {
+      "completion_length": 61.15390832424164,
+      "epoch": 0.185522610568163,
+      "grad_norm": 1.655745220212549,
+      "kl": 1.127587890625,
+      "learning_rate": 1.9563499025108e-05,
+      "loss": 0.0451,
+      "reward": 1.0997396193444728,
+      "reward_std": 0.19142945660278202,
+      "rewards/accuracy_reward": 0.15260417079553007,
+      "rewards/format_reward": 0.9471354357898235,
+      "step": 70
+    },
+    {
+      "completion_length": 72.45364763736725,
+      "epoch": 0.19877422560874605,
+      "grad_norm": 1.5309384490003093,
+      "kl": 1.916259765625,
+      "learning_rate": 1.941787727566613e-05,
+      "loss": 0.0768,
+      "reward": 1.1015625432133676,
+      "reward_std": 0.18512018411420286,
+      "rewards/accuracy_reward": 0.1510416709119454,
+      "rewards/format_reward": 0.9505208522081375,
+      "step": 75
+    },
+    {
+      "completion_length": 63.51927285194397,
+      "epoch": 0.21202584064932914,
+      "grad_norm": 44.22069024914294,
+      "kl": 13.19677734375,
+      "learning_rate": 1.9252038574264403e-05,
+      "loss": 0.5275,
+      "reward": 1.1307292066514492,
+      "reward_std": 0.17913860315456986,
+      "rewards/accuracy_reward": 0.17135417100507766,
+      "rewards/format_reward": 0.9593750186264515,
+      "step": 80
+    },
+    {
+      "completion_length": 36.28333444595337,
+      "epoch": 0.2252774556899122,
+      "grad_norm": 1.3942674418927083,
+      "kl": 1.52724609375,
+      "learning_rate": 1.90663389197082e-05,
+      "loss": 0.0612,
+      "reward": 1.1416667088866235,
+      "reward_std": 0.17510308041237294,
+      "rewards/accuracy_reward": 0.1731770876329392,
+      "rewards/format_reward": 0.9684896029531955,
+      "step": 85
+    },
+    {
+      "completion_length": 35.96145946979523,
+      "epoch": 0.23852907073049529,
+      "grad_norm": 0.735877152054416,
+      "kl": 0.802783203125,
+      "learning_rate": 1.8861176945456542e-05,
+      "loss": 0.0321,
+      "reward": 1.1364583753049373,
+      "reward_std": 0.14004945596680046,
+      "rewards/accuracy_reward": 0.1510416712379083,
+      "rewards/format_reward": 0.985416678339243,
+      "step": 90
+    },
+    {
+      "completion_length": 112.99114933013917,
+      "epoch": 0.25178068577107837,
+      "grad_norm": 118.15953837921309,
+      "kl": 1.56982421875,
+      "learning_rate": 1.8636993063892822e-05,
+      "loss": 0.0628,
+      "reward": 1.093229205161333,
+      "reward_std": 0.20341236842796206,
+      "rewards/accuracy_reward": 0.1388020883779973,
+      "rewards/format_reward": 0.9544271007180214,
+      "step": 95
+    },
+    {
+      "completion_length": 82.72708551883697,
+      "epoch": 0.2650323008116614,
+      "grad_norm": 0.6503495413677531,
+      "kl": 1.816162109375,
+      "learning_rate": 1.8394268520910467e-05,
+      "loss": 0.0729,
+      "reward": 1.106510452926159,
+      "reward_std": 0.18918234938755632,
+      "rewards/accuracy_reward": 0.14531250444706528,
+      "rewards/format_reward": 0.9611979350447655,
+      "step": 100
+    },
+    {
+      "epoch": 0.2650323008116614,
+      "eval_completion_length": 67.22291870117188,
+      "eval_kl": 0.95859375,
+      "eval_loss": 0.03748759999871254,
+      "eval_reward": 1.1604167222976685,
+      "eval_reward_std": 0.15603546500205995,
+      "eval_rewards/accuracy_reward": 0.18958334028720855,
+      "eval_rewards/format_reward": 0.9708333492279053,
+      "eval_runtime": 37.1259,
+      "eval_samples_per_second": 2.667,
+      "eval_steps_per_second": 0.054,
+      "step": 100
+    },
+    {
+      "completion_length": 66.1372414112091,
+      "epoch": 0.2782839158522445,
+      "grad_norm": 1.3755764716936518,
+      "kl": 0.9490234375,
+      "learning_rate": 1.8133524362843105e-05,
+      "loss": 0.0379,
+      "reward": 1.1291667006909847,
+      "reward_std": 0.18105815881863235,
+      "rewards/accuracy_reward": 0.1591145883547142,
+      "rewards/format_reward": 0.9700521022081375,
+      "step": 105
+    },
+    {
+      "completion_length": 59.52109546661377,
+      "epoch": 0.2915355308928276,
+      "grad_norm": 0.9209499737780213,
+      "kl": 0.947314453125,
+      "learning_rate": 1.7855320317956785e-05,
+      "loss": 0.0379,
+      "reward": 1.1583333738148212,
+      "reward_std": 0.16386113320477308,
+      "rewards/accuracy_reward": 0.1846354216337204,
+      "rewards/format_reward": 0.9736979328095913,
+      "step": 110
+    },
+    {
+      "completion_length": 63.279689526557924,
+      "epoch": 0.3047871459334106,
+      "grad_norm": 2.0782043783063098,
+      "kl": 1.91630859375,
+      "learning_rate": 1.7560253594905425e-05,
+      "loss": 0.0766,
+      "reward": 1.0916667006909848,
+      "reward_std": 0.22827091701328756,
+      "rewards/accuracy_reward": 0.1575520870508626,
+      "rewards/format_reward": 0.9341145999729633,
+      "step": 115
+    },
+    {
+      "completion_length": 45.06198041439056,
+      "epoch": 0.3180387609739937,
+      "grad_norm": 7.3972784312018165,
+      "kl": 1.8796875,
+      "learning_rate": 1.7248957600728664e-05,
+      "loss": 0.0752,
+      "reward": 1.1247396238148213,
+      "reward_std": 0.21942297443747522,
+      "rewards/accuracy_reward": 0.1770833388902247,
+      "rewards/format_reward": 0.9476562723517418,
+      "step": 120
+    },
+    {
+      "completion_length": 57.739324641227725,
+      "epoch": 0.3312903760145768,
+      "grad_norm": 111.62950854973025,
+      "kl": 2.24365234375,
+      "learning_rate": 1.6922100581144228e-05,
+      "loss": 0.0898,
+      "reward": 1.133333370089531,
+      "reward_std": 0.19298880095593632,
+      "rewards/accuracy_reward": 0.18072917175013573,
+      "rewards/format_reward": 0.9526041865348815,
+      "step": 125
+    },
+    {
+      "completion_length": 59.59635586738587,
+      "epoch": 0.34454199105515987,
+      "grad_norm": 42.715441871558575,
+      "kl": 2.253076171875,
+      "learning_rate": 1.658038418605361e-05,
+      "loss": 0.0902,
+      "reward": 1.1502604573965072,
+      "reward_std": 0.17992896740324796,
+      "rewards/accuracy_reward": 0.18854167202953248,
+      "rewards/format_reward": 0.9617187701165676,
+      "step": 130
+    },
+    {
+      "completion_length": 102.57187860012054,
+      "epoch": 0.3577936060957429,
+      "grad_norm": 1.516051179569486,
+      "kl": 6.09853515625,
+      "learning_rate": 1.6224541963340392e-05,
+      "loss": 0.2434,
+      "reward": 1.085937537252903,
+      "reward_std": 0.24600069066509606,
+      "rewards/accuracy_reward": 0.16536458879709243,
+      "rewards/format_reward": 0.9205729350447655,
+      "step": 135
+    },
+    {
+      "completion_length": 70.24948139190674,
+      "epoch": 0.371045221136326,
+      "grad_norm": 0.6833587074248292,
+      "kl": 1.281982421875,
+      "learning_rate": 1.5855337784194576e-05,
+      "loss": 0.0513,
+      "reward": 1.130468787252903,
+      "reward_std": 0.1880698367021978,
+      "rewards/accuracy_reward": 0.1742187554948032,
+      "rewards/format_reward": 0.9562500216066837,
+      "step": 140
+    },
+    {
+      "completion_length": 51.64739756584167,
+      "epoch": 0.3842968361769091,
+      "grad_norm": 0.5876883269514013,
+      "kl": 0.767578125,
+      "learning_rate": 1.5473564203343173e-05,
+      "loss": 0.0307,
+      "reward": 1.1697917066514492,
+      "reward_std": 0.14846973307430744,
+      "rewards/accuracy_reward": 0.194270838634111,
+      "rewards/format_reward": 0.9755208499729633,
+      "step": 145
+    },
+    {
+      "completion_length": 73.53776233196258,
+      "epoch": 0.3975484512174921,
+      "grad_norm": 0.7747226744574793,
+      "kl": 1.310546875,
+      "learning_rate": 1.5080040757707045e-05,
+      "loss": 0.0524,
+      "reward": 1.1518229566514493,
+      "reward_std": 0.200691845593974,
+      "rewards/accuracy_reward": 0.19609375507570803,
+      "rewards/format_reward": 0.9557291880249977,
+      "step": 150
+    },
+    {
+      "completion_length": 54.49192841053009,
+      "epoch": 0.4108000662580752,
+      "grad_norm": 1.8482541426323007,
+      "kl": 1.24716796875,
+      "learning_rate": 1.4675612207136283e-05,
+      "loss": 0.0499,
+      "reward": 1.1692708671092986,
+      "reward_std": 0.18913309550844132,
+      "rewards/accuracy_reward": 0.19973958930931984,
+      "rewards/format_reward": 0.9695312656462193,
+      "step": 155
+    },
+    {
+      "completion_length": 63.277345180511475,
+      "epoch": 0.4240516812986583,
+      "grad_norm": 0.7266536465127268,
+      "kl": 1.099462890625,
+      "learning_rate": 1.4261146721000554e-05,
+      "loss": 0.044,
+      "reward": 1.1781250417232514,
+      "reward_std": 0.18760176608338952,
+      "rewards/accuracy_reward": 0.2104166726814583,
+      "rewards/format_reward": 0.9677083566784859,
+      "step": 160
+    },
+    {
+      "completion_length": 75.79791910648346,
+      "epoch": 0.43730329633924137,
+      "grad_norm": 2.1758113794869813,
+      "kl": 1.234765625,
+      "learning_rate": 1.3837534014527292e-05,
+      "loss": 0.0494,
+      "reward": 1.161197952926159,
+      "reward_std": 0.21902822507545353,
+      "rewards/accuracy_reward": 0.2101562550291419,
+      "rewards/format_reward": 0.9510416895151138,
+      "step": 165
+    },
+    {
+      "completion_length": 66.98359591960907,
+      "epoch": 0.4505549113798244,
+      "grad_norm": 0.9764655215345127,
+      "kl": 1.21435546875,
+      "learning_rate": 1.3405683438888281e-05,
+      "loss": 0.0486,
+      "reward": 1.156770868599415,
+      "reward_std": 0.19157032966613768,
+      "rewards/accuracy_reward": 0.19843750577419997,
+      "rewards/format_reward": 0.9583333522081375,
+      "step": 170
+    },
+    {
+      "completion_length": 71.22474150657654,
+      "epoch": 0.4638065264204075,
+      "grad_norm": 2.8758489653544426,
+      "kl": 1.022119140625,
+      "learning_rate": 1.2966522029134622e-05,
+      "loss": 0.0409,
+      "reward": 1.1705729521811008,
+      "reward_std": 0.20062823588959872,
+      "rewards/accuracy_reward": 0.2088541722856462,
+      "rewards/format_reward": 0.9617187708616257,
+      "step": 175
+    },
+    {
+      "completion_length": 79.03854382038116,
+      "epoch": 0.47705814146099057,
+      "grad_norm": 0.9283264048838794,
+      "kl": 0.855029296875,
+      "learning_rate": 1.252099251417048e-05,
+      "loss": 0.0342,
+      "reward": 1.1533854559063912,
+      "reward_std": 0.21582746594212948,
+      "rewards/accuracy_reward": 0.19791667133104057,
+      "rewards/format_reward": 0.9554687716066838,
+      "step": 180
+    },
+    {
+      "completion_length": 66.87500190734863,
+      "epoch": 0.49030975650157366,
+      "grad_norm": 0.5205596072083288,
+      "kl": 0.81142578125,
+      "learning_rate": 1.2070051293037493e-05,
+      "loss": 0.0325,
+      "reward": 1.139843787997961,
+      "reward_std": 0.18398637240752577,
+      "rewards/accuracy_reward": 0.1703125050989911,
+      "rewards/format_reward": 0.9695312649011611,
+      "step": 185
+    },
+    {
+      "completion_length": 67.04583523273467,
+      "epoch": 0.5035613715421567,
+      "grad_norm": 0.6361974079864325,
+      "kl": 0.906689453125,
+      "learning_rate": 1.1614666381854107e-05,
+      "loss": 0.0363,
+      "reward": 1.1380208715796472,
+      "reward_std": 0.19017330487258732,
+      "rewards/accuracy_reward": 0.1723958380986005,
+      "rewards/format_reward": 0.9656250178813934,
+      "step": 190
+    },
+    {
+      "completion_length": 78.92995023727417,
+      "epoch": 0.5168129865827398,
+      "grad_norm": 1.891991619936415,
+      "kl": 0.98994140625,
+      "learning_rate": 1.115581533581701e-05,
+      "loss": 0.0396,
+      "reward": 1.148697953671217,
+      "reward_std": 0.20872723800130188,
+      "rewards/accuracy_reward": 0.19114583905320615,
+      "rewards/format_reward": 0.9575521036982536,
+      "step": 195
+    },
+    {
+      "completion_length": 62.139325046539305,
+      "epoch": 0.5300646016233228,
+      "grad_norm": 1.0280148057118665,
+      "kl": 0.980517578125,
+      "learning_rate": 1.0694483150725458e-05,
+      "loss": 0.0392,
+      "reward": 1.1401042118668556,
+      "reward_std": 0.18719495739787817,
+      "rewards/accuracy_reward": 0.1739583388902247,
+      "rewards/format_reward": 0.9661458522081375,
+      "step": 200
+    },
+    {
+      "epoch": 0.5300646016233228,
+      "eval_completion_length": 40.61458511352539,
+      "eval_kl": 0.809375,
+      "eval_loss": 0.03229926526546478,
+      "eval_reward": 1.1958333730697632,
+      "eval_reward_std": 0.2178212285041809,
+      "eval_rewards/accuracy_reward": 0.22500000894069672,
+      "eval_rewards/format_reward": 0.9708333492279053,
+      "eval_runtime": 26.7551,
+      "eval_samples_per_second": 3.7,
+      "eval_steps_per_second": 0.075,
+      "step": 200
+    },
+    {
+      "completion_length": 74.88411645889282,
+      "epoch": 0.543316216663906,
+      "grad_norm": 2.9731936983758094,
+      "kl": 1.171484375,
+      "learning_rate": 1.0231660148533183e-05,
+      "loss": 0.0469,
+      "reward": 1.135937536507845,
+      "reward_std": 0.21834916472434998,
+      "rewards/accuracy_reward": 0.18307292230892927,
+      "rewards/format_reward": 0.9528646014630795,
+      "step": 205
+    },
+    {
+      "completion_length": 73.33125195503234,
+      "epoch": 0.556567831704489,
+      "grad_norm": 0.742765288728699,
+      "kl": 1.08818359375,
+      "learning_rate": 9.768339851466818e-06,
+      "loss": 0.0435,
+      "reward": 1.1684896178543567,
+      "reward_std": 0.21213787337765097,
+      "rewards/accuracy_reward": 0.21119792191311718,
+      "rewards/format_reward": 0.9572916842997075,
+      "step": 210
+    },
+    {
+      "completion_length": 51.361980509758,
+      "epoch": 0.569819446745072,
+      "grad_norm": 0.5296161980646436,
+      "kl": 0.91787109375,
+      "learning_rate": 9.305516849274542e-06,
+      "loss": 0.0367,
+      "reward": 1.1898437894880771,
+      "reward_std": 0.15477278176695108,
+      "rewards/accuracy_reward": 0.20937500728759914,
+      "rewards/format_reward": 0.9804687649011612,
+      "step": 215
+    },
+    {
+      "completion_length": 66.18489775657653,
+      "epoch": 0.5830710617856552,
+      "grad_norm": 1.319642176578017,
+      "kl": 1.52646484375,
+      "learning_rate": 8.844184664182993e-06,
+      "loss": 0.061,
+      "reward": 1.1859375417232514,
+      "reward_std": 0.1461514844559133,
+      "rewards/accuracy_reward": 0.2041666724253446,
+      "rewards/format_reward": 0.9817708477377891,
+      "step": 220
+    },
+    {
+      "completion_length": 74.04218964576721,
+      "epoch": 0.5963226768262382,
+      "grad_norm": 0.5794834549826069,
+      "kl": 0.8259765625,
+      "learning_rate": 8.385333618145896e-06,
+      "loss": 0.033,
+      "reward": 1.1812500379979611,
+      "reward_std": 0.17319163950160146,
+      "rewards/accuracy_reward": 0.2057291725417599,
+      "rewards/format_reward": 0.9755208514630794,
+      "step": 225
+    },
+    {
+      "completion_length": 91.75937752723694,
+      "epoch": 0.6095742918668212,
+      "grad_norm": 0.6342381554755581,
+      "kl": 0.9462890625,
+      "learning_rate": 7.929948706962508e-06,
+      "loss": 0.0379,
+      "reward": 1.172135455161333,
+      "reward_std": 0.20429653744213283,
+      "rewards/accuracy_reward": 0.20885417321696878,
+      "rewards/format_reward": 0.9632812686264515,
+      "step": 230
+    },
+    {
+      "completion_length": 72.45026273727417,
+      "epoch": 0.6228259069074044,
+      "grad_norm": 0.6073134707677167,
+      "kl": 0.919384765625,
+      "learning_rate": 7.479007485829523e-06,
+      "loss": 0.0368,
+      "reward": 1.1729167103767395,
+      "reward_std": 0.19189335247501732,
+      "rewards/accuracy_reward": 0.2078125061932951,
+      "rewards/format_reward": 0.9651041857898235,
+      "step": 235
+    },
+    {
+      "completion_length": 62.588804078102115,
+      "epoch": 0.6360775219479874,
+      "grad_norm": 0.7005504700359295,
+      "kl": 0.944482421875,
+      "learning_rate": 7.033477970865381e-06,
+      "loss": 0.0378,
+      "reward": 1.1804687909781932,
+      "reward_std": 0.18590812091715633,
+      "rewards/accuracy_reward": 0.2109375050989911,
+      "rewards/format_reward": 0.9695312678813934,
+      "step": 240
+    },
+    {
+      "completion_length": 65.3341163635254,
+      "epoch": 0.6493291369885705,
+      "grad_norm": 0.849514155518688,
+      "kl": 0.924365234375,
+      "learning_rate": 6.5943165611117244e-06,
+      "loss": 0.037,
+      "reward": 1.2031250335276127,
+      "reward_std": 0.1887207916006446,
+      "rewards/accuracy_reward": 0.23307292351964862,
+      "rewards/format_reward": 0.9700520999729634,
+      "step": 245
+    },
+    {
+      "completion_length": 69.82161645889282,
+      "epoch": 0.6625807520291536,
+      "grad_norm": 0.46246316120022374,
+      "kl": 0.884033203125,
+      "learning_rate": 6.16246598547271e-06,
+      "loss": 0.0353,
+      "reward": 1.1966146215796472,
+      "reward_std": 0.19367394060827792,
+      "rewards/accuracy_reward": 0.2286458395421505,
+      "rewards/format_reward": 0.9679687686264515,
+      "step": 250
+    },
+    {
+      "completion_length": 69.94323103427887,
+      "epoch": 0.6758323670697366,
+      "grad_norm": 0.5407319977607519,
+      "kl": 0.885986328125,
+      "learning_rate": 5.7388532789994476e-06,
+      "loss": 0.0354,
+      "reward": 1.1953125350177287,
+      "reward_std": 0.18934729555621743,
+      "rewards/accuracy_reward": 0.22708333991467952,
+      "rewards/format_reward": 0.9682291880249977,
+      "step": 255
+    },
+    {
+      "completion_length": 66.55468945503235,
+      "epoch": 0.6890839821103197,
+      "grad_norm": 1.1266711173773165,
+      "kl": 0.86923828125,
+      "learning_rate": 5.324387792863719e-06,
+      "loss": 0.0348,
+      "reward": 1.2169271171092988,
+      "reward_std": 0.18687426932156087,
+      "rewards/accuracy_reward": 0.24583334014751018,
+      "rewards/format_reward": 0.9710937678813935,
+      "step": 260
+    },
+    {
+      "completion_length": 60.47448055744171,
+      "epoch": 0.7023355971509028,
+      "grad_norm": 1.5540794272212215,
+      "kl": 12.859765625,
+      "learning_rate": 4.9199592422929545e-06,
+      "loss": 0.5124,
+      "reward": 1.2091146230697631,
+      "reward_std": 0.18144828863441945,
+      "rewards/accuracy_reward": 0.23463542396202683,
+      "rewards/format_reward": 0.9744791835546494,
+      "step": 265
+    },
+    {
+      "completion_length": 64.92838711738587,
+      "epoch": 0.7155872121914858,
+      "grad_norm": 3.134764657649563,
+      "kl": 0.9081787109375,
+      "learning_rate": 4.52643579665683e-06,
+      "loss": 0.0363,
+      "reward": 1.1888021185994149,
+      "reward_std": 0.18178390581160783,
+      "rewards/accuracy_reward": 0.22135417270474136,
+      "rewards/format_reward": 0.9674479372799396,
+      "step": 270
+    },
+    {
+      "completion_length": 65.88958518505096,
+      "epoch": 0.7288388272320689,
+      "grad_norm": 0.5517576711381919,
+      "kl": 0.887744140625,
+      "learning_rate": 4.144662215805426e-06,
+      "loss": 0.0355,
+      "reward": 1.1924479484558106,
+      "reward_std": 0.20131790162995458,
+      "rewards/accuracy_reward": 0.22864583949558437,
+      "rewards/format_reward": 0.9638021029531956,
+      "step": 275
+    },
+    {
+      "completion_length": 82.00130467414856,
+      "epoch": 0.742090442272652,
+      "grad_norm": 0.6633430041489747,
+      "kl": 0.97236328125,
+      "learning_rate": 3.7754580366596116e-06,
+      "loss": 0.0389,
+      "reward": 1.1841146223247052,
+      "reward_std": 0.22335054534487425,
+      "rewards/accuracy_reward": 0.2317708389600739,
+      "rewards/format_reward": 0.9523437701165676,
+      "step": 280
+    },
+    {
+      "completion_length": 74.26015827655792,
+      "epoch": 0.755342057313235,
+      "grad_norm": 0.574268931795741,
+      "kl": 0.999267578125,
+      "learning_rate": 3.419615813946392e-06,
+      "loss": 0.04,
+      "reward": 1.1713542021811008,
+      "reward_std": 0.2193179450929165,
+      "rewards/accuracy_reward": 0.2143229231936857,
+      "rewards/format_reward": 0.9570312723517418,
+      "step": 285
+    },
+    {
+      "completion_length": 74.4083356142044,
+      "epoch": 0.7685936723538181,
+      "grad_norm": 5.33936842764644,
+      "kl": 1.0833984375,
+      "learning_rate": 3.0778994188557722e-06,
+      "loss": 0.0433,
+      "reward": 1.165364620089531,
+      "reward_std": 0.22240204717963935,
+      "rewards/accuracy_reward": 0.20937500593718142,
+      "rewards/format_reward": 0.9559895992279053,
+      "step": 290
+    },
+    {
+      "completion_length": 69.42942883968354,
+      "epoch": 0.7818452873944012,
+      "grad_norm": 0.5719187957247389,
+      "kl": 0.88525390625,
+      "learning_rate": 2.7510423992713374e-06,
+      "loss": 0.0354,
+      "reward": 1.1901042066514491,
+      "reward_std": 0.20185629166662694,
+      "rewards/accuracy_reward": 0.2296875052852556,
+      "rewards/format_reward": 0.9604166880249977,
+      "step": 295
+    },
+    {
+      "completion_length": 65.15078330039978,
+      "epoch": 0.7950969024349842,
+      "grad_norm": 0.6931319151935165,
+      "kl": 0.8555908203125,
+      "learning_rate": 2.4397464050945753e-06,
+      "loss": 0.0342,
+      "reward": 1.183593788743019,
+      "reward_std": 0.18570943493396044,
+      "rewards/accuracy_reward": 0.21953125703148543,
+      "rewards/format_reward": 0.9640625230967999,
+      "step": 300
+    },
+    {
+      "epoch": 0.7950969024349842,
+      "eval_completion_length": 131.92916946411134,
+      "eval_kl": 0.98359375,
+      "eval_loss": 0.04183487966656685,
+      "eval_reward": 1.225000023841858,
+      "eval_reward_std": 0.22099787294864653,
+      "eval_rewards/accuracy_reward": 0.27291667759418486,
+      "eval_rewards/format_reward": 0.9520833492279053,
+      "eval_runtime": 36.5234,
+      "eval_samples_per_second": 2.711,
+      "eval_steps_per_second": 0.055,
+      "step": 300
+    },
+    {
+      "completion_length": 78.14088757038117,
+      "epoch": 0.8083485174755674,
+      "grad_norm": 0.6675857214490832,
+      "kl": 0.90390625,
+      "learning_rate": 2.144679682043217e-06,
+      "loss": 0.0362,
+      "reward": 1.1669271178543568,
+      "reward_std": 0.21354180253110827,
+      "rewards/accuracy_reward": 0.21666667249519378,
+      "rewards/format_reward": 0.9502604372799397,
+      "step": 305
+    },
+    {
+      "completion_length": 75.2903666973114,
+      "epoch": 0.8216001325161504,
+      "grad_norm": 0.5712805830429172,
+      "kl": 0.9865234375,
+      "learning_rate": 1.8664756371568981e-06,
+      "loss": 0.0394,
+      "reward": 1.2026041999459267,
+      "reward_std": 0.2196099391207099,
+      "rewards/accuracy_reward": 0.24973958965856582,
+      "rewards/format_reward": 0.9528646007180214,
+      "step": 310
+    },
+    {
+      "completion_length": 65.9781269788742,
+      "epoch": 0.8348517475567335,
+      "grad_norm": 0.5628544528150197,
+      "kl": 0.91279296875,
+      "learning_rate": 1.605731479089534e-06,
+      "loss": 0.0365,
+      "reward": 1.1934896178543568,
+      "reward_std": 0.20774728069081902,
+      "rewards/accuracy_reward": 0.2325520909158513,
+      "rewards/format_reward": 0.9609375163912773,
+      "step": 315
+    },
+    {
+      "completion_length": 65.84948132038116,
+      "epoch": 0.8481033625973166,
+      "grad_norm": 0.6078831877141712,
+      "kl": 0.885400390625,
+      "learning_rate": 1.363006936107183e-06,
+      "loss": 0.0354,
+      "reward": 1.2033854566514492,
+      "reward_std": 0.19603493059985339,
+      "rewards/accuracy_reward": 0.24140625675208865,
+      "rewards/format_reward": 0.9619791828095913,
+      "step": 320
+    },
+    {
+      "completion_length": 72.75000171661377,
+      "epoch": 0.8613549776378996,
+      "grad_norm": 0.6931019839903174,
+      "kl": 0.831005859375,
+      "learning_rate": 1.1388230545434652e-06,
+      "loss": 0.0332,
+      "reward": 1.1809896253049375,
+      "reward_std": 0.2047369409352541,
+      "rewards/accuracy_reward": 0.2205729227978736,
+      "rewards/format_reward": 0.9604166850447655,
+      "step": 325
+    },
+    {
+      "completion_length": 71.47682478427888,
+      "epoch": 0.8746065926784827,
+      "grad_norm": 0.588939126765295,
+      "kl": 0.87451171875,
+      "learning_rate": 9.336610802918044e-07,
+      "loss": 0.035,
+      "reward": 1.2072917088866233,
+      "reward_std": 0.20571860703639686,
+      "rewards/accuracy_reward": 0.24713542452082038,
+      "rewards/format_reward": 0.9601562701165676,
+      "step": 330
+    },
+    {
+      "completion_length": 74.76771051883698,
+      "epoch": 0.8878582077190658,
+      "grad_norm": 0.6165305150608602,
+      "kl": 0.91201171875,
+      "learning_rate": 7.479614257355972e-07,
+      "loss": 0.0365,
+      "reward": 1.193750037997961,
+      "reward_std": 0.2028288107831031,
+      "rewards/accuracy_reward": 0.23385417191311716,
+      "rewards/format_reward": 0.9598958544433117,
+      "step": 335
+    },
+    {
+      "completion_length": 75.77942934036255,
+      "epoch": 0.9011098227596488,
+      "grad_norm": 0.6956738727145818,
+      "kl": 0.98857421875,
+      "learning_rate": 5.821227243338712e-07,
+      "loss": 0.0395,
+      "reward": 1.2203125365078449,
+      "reward_std": 0.20734485271386802,
+      "rewards/accuracy_reward": 0.25651042377576233,
+      "rewards/format_reward": 0.9638021044433117,
+      "step": 340
+    },
+    {
+      "completion_length": 76.83776240348816,
+      "epoch": 0.9143614378002319,
+      "grad_norm": 0.5715844965797664,
+      "kl": 0.936279296875,
+      "learning_rate": 4.3650097489200125e-07,
+      "loss": 0.0375,
+      "reward": 1.186979205161333,
+      "reward_std": 0.20779294366948306,
+      "rewards/accuracy_reward": 0.2270833399379626,
+      "rewards/format_reward": 0.9598958536982536,
+      "step": 345
+    },
+    {
+      "completion_length": 70.03672063350677,
+      "epoch": 0.927613052840815,
+      "grad_norm": 1.1274397177481879,
+      "kl": 0.891259765625,
+      "learning_rate": 3.114087773543939e-07,
+      "loss": 0.0356,
+      "reward": 1.224218787252903,
+      "reward_std": 0.19236171743832528,
+      "rewards/accuracy_reward": 0.26041667265817525,
+      "rewards/format_reward": 0.9638021029531956,
+      "step": 350
+    },
+    {
+      "completion_length": 73.23906478881835,
+      "epoch": 0.940864667881398,
+      "grad_norm": 0.7990922943480933,
+      "kl": 0.96572265625,
+      "learning_rate": 2.071146617596276e-07,
+      "loss": 0.0386,
+      "reward": 1.1976562894880771,
+      "reward_std": 0.199229073850438,
+      "rewards/accuracy_reward": 0.23359375761356205,
+      "rewards/format_reward": 0.9640625216066837,
+      "step": 355
+    },
+    {
+      "completion_length": 69.15703308582306,
+      "epoch": 0.9541162829219811,
+      "grad_norm": 0.509865467948569,
+      "kl": 0.85087890625,
+      "learning_rate": 1.2384251179857642e-07,
+      "loss": 0.034,
+      "reward": 1.2054687939584254,
+      "reward_std": 0.20947544923983513,
+      "rewards/accuracy_reward": 0.24348958916962146,
+      "rewards/format_reward": 0.9619791857898236,
+      "step": 360
+    },
+    {
+      "completion_length": 71.84192910194398,
+      "epoch": 0.9673678979625642,
+      "grad_norm": 0.6201257537439341,
+      "kl": 0.922705078125,
+      "learning_rate": 6.177108421292266e-08,
+      "loss": 0.0369,
+      "reward": 1.2122396215796472,
+      "reward_std": 0.20185307771898806,
+      "rewards/accuracy_reward": 0.24661458991467952,
+      "rewards/format_reward": 0.9656250216066837,
+      "step": 365
+    },
+    {
+      "completion_length": 71.96797089576721,
+      "epoch": 0.9806195130031473,
+      "grad_norm": 0.5354564912646672,
+      "kl": 0.902490234375,
+      "learning_rate": 2.1033625065747244e-08,
+      "loss": 0.0361,
+      "reward": 1.1966146215796472,
+      "reward_std": 0.20819322955794634,
+      "rewards/accuracy_reward": 0.23541667219251394,
+      "rewards/format_reward": 0.9611979335546493,
+      "step": 370
+    },
+    {
+      "completion_length": 72.72734587192535,
+      "epoch": 0.9938711280437303,
+      "grad_norm": 0.7792560018707632,
+      "kl": 0.894091796875,
+      "learning_rate": 1.7175837079452806e-09,
+      "loss": 0.0358,
+      "reward": 1.207812535762787,
+      "reward_std": 0.2039037857670337,
+      "rewards/accuracy_reward": 0.24661458996124566,
+      "rewards/format_reward": 0.9611979350447655,
+      "step": 375
+    },
+    {
+      "completion_length": 76.10937750339508,
+      "epoch": 0.9991717740599636,
+      "kl": 0.9320068359375,
+      "reward": 1.1835937853902578,
+      "reward_std": 0.20871941454242915,
+      "rewards/accuracy_reward": 0.21940104698296636,
+      "rewards/format_reward": 0.9641927257180214,
+      "step": 377,
+      "total_flos": 0.0,
+      "train_loss": 0.05497839203927236,
+      "train_runtime": 37424.3352,
+      "train_samples_per_second": 1.936,
+      "train_steps_per_second": 0.01
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 377,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d41d8074a721706e28ebb3052c3de5a5beaca4a587399980f6fcc9b79058a58
+size 7544