Upload 13 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,208 @@
----
-license: apache-2.0
----
+---
+base_model: unsloth/gemma-3n-e4b-it-unsloth-bnb-4bit
+library_name: peft
+tags:
+- base_model:adapter:unsloth/gemma-3n-e4b-it-unsloth-bnb-4bit
+- lora
+- orpo
+- trl
+- unsloth
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "PeftModel",
+    "parent_library": "peft.peft_model"
+  },
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": null,
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b51c92908e9d4367dde0508d96b1009f190249632eb52b7f3e65509c9b2d1c61
+size 321613536

chat_template.jinja

@@ -0,0 +1,49 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'audio' -%}
+                {{ '<audio_soft_token>' }}
+            {%- elif item['type'] == 'image' -%}
+                {{ '<image_soft_token>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8917fdde758de8a1eec9551dce7e4363aab56dd16718cf651912758050fcf89f
+size 14645

scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35e9ad20d4893f9f01182366cdf6ad0012a1e3a62a72938d4b700be72801692f
+size 1383

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce43d0d55120f6da2d9f1d2bc090c52fb467b83eac8a52ca4c1fd67161b1d91
+size 1465

special_tokens_map.json

@@ -0,0 +1,36 @@
+{
+  "audio_token": "<audio_soft_token>",
+  "boa_token": "<start_of_audio>",
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoa_token": "<end_of_audio>",
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<end_of_turn>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c35ee648c07754b44cd9e371c75d4caa05c4504910b7ad29b1847ee9d8ba5d
+size 33442553

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea5f0cc48abfbfc04d14562270a32e02149a3e7035f368cc5a462786f4a59961
+size 4696020

trainer_state.json

@@ -0,0 +1,1834 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.19704433497536947,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0019704433497536944,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -1.4796818494796753,
+      "log_odds_ratio": -1.8135762214660645,
+      "logits/chosen": -25.554729461669922,
+      "logits/rejected": -25.793912887573242,
+      "logps/chosen": -14.237062454223633,
+      "logps/rejected": -12.757383346557617,
+      "loss": 14.4972,
+      "nll_loss": 14.315820693969727,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.4237061738967896,
+      "rewards/margins": -0.14796781539916992,
+      "rewards/rejected": -1.2757384777069092,
+      "step": 1
+    },
+    {
+      "epoch": 0.003940886699507389,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -1.5265605449676514,
+      "log_odds_ratio": -2.005342721939087,
+      "logits/chosen": -23.914981842041016,
+      "logits/rejected": -24.484413146972656,
+      "logps/chosen": -13.575830459594727,
+      "logps/rejected": -12.049341201782227,
+      "loss": 14.5969,
+      "nll_loss": 14.396320343017578,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.3575830459594727,
+      "rewards/margins": -0.1526489555835724,
+      "rewards/rejected": -1.2049341201782227,
+      "step": 2
+    },
+    {
+      "epoch": 0.005911330049261084,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -1.880261778831482,
+      "log_odds_ratio": -2.292414665222168,
+      "logits/chosen": -24.909160614013672,
+      "logits/rejected": -25.134037017822266,
+      "logps/chosen": -15.220357894897461,
+      "logps/rejected": -13.340100288391113,
+      "loss": 14.9211,
+      "nll_loss": 14.691848754882812,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.522035837173462,
+      "rewards/margins": -0.1880258023738861,
+      "rewards/rejected": -1.334010124206543,
+      "step": 3
+    },
+    {
+      "epoch": 0.007881773399014778,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -5.7012481689453125,
+      "log_odds_ratio": -5.88106107711792,
+      "logits/chosen": -25.190185546875,
+      "logits/rejected": -25.323806762695312,
+      "logps/chosen": -17.655122756958008,
+      "logps/rejected": -11.953886032104492,
+      "loss": 15.1497,
+      "nll_loss": 14.561641693115234,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.7655123472213745,
+      "rewards/margins": -0.5701237916946411,
+      "rewards/rejected": -1.1953885555267334,
+      "step": 4
+    },
+    {
+      "epoch": 0.009852216748768473,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": 0.7353910803794861,
+      "log_odds_ratio": -0.5241638422012329,
+      "logits/chosen": -24.11300277709961,
+      "logits/rejected": -24.172021865844727,
+      "logps/chosen": -11.486623764038086,
+      "logps/rejected": -12.222051620483398,
+      "loss": 14.4485,
+      "nll_loss": 14.396095275878906,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1486624479293823,
+      "rewards/margins": 0.07354274392127991,
+      "rewards/rejected": -1.2222051620483398,
+      "step": 5
+    },
+    {
+      "epoch": 0.011822660098522168,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -0.8240797519683838,
+      "log_odds_ratio": -1.3969202041625977,
+      "logits/chosen": -24.877756118774414,
+      "logits/rejected": -25.370180130004883,
+      "logps/chosen": -14.42114543914795,
+      "logps/rejected": -13.597131729125977,
+      "loss": 14.2763,
+      "nll_loss": 14.136587142944336,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.4421145915985107,
+      "rewards/margins": -0.08240137994289398,
+      "rewards/rejected": -1.3597133159637451,
+      "step": 6
+    },
+    {
+      "epoch": 0.013793103448275862,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -3.008707046508789,
+      "log_odds_ratio": -3.6310629844665527,
+      "logits/chosen": -24.327289581298828,
+      "logits/rejected": -24.47004508972168,
+      "logps/chosen": -15.620328903198242,
+      "logps/rejected": -12.611661911010742,
+      "loss": 14.8814,
+      "nll_loss": 14.518259048461914,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.5620328187942505,
+      "rewards/margins": -0.30086660385131836,
+      "rewards/rejected": -1.2611662149429321,
+      "step": 7
+    },
+    {
+      "epoch": 0.015763546798029555,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -3.6951146125793457,
+      "log_odds_ratio": -3.8078808784484863,
+      "logits/chosen": -24.65069580078125,
+      "logits/rejected": -24.482654571533203,
+      "logps/chosen": -14.109657287597656,
+      "logps/rejected": -10.4146728515625,
+      "loss": 14.7568,
+      "nll_loss": 14.375982284545898,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -1.4109658002853394,
+      "rewards/margins": -0.36949849128723145,
+      "rewards/rejected": -1.041467308998108,
+      "step": 8
+    },
+    {
+      "epoch": 0.017733990147783252,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": 0.22909605503082275,
+      "log_odds_ratio": -1.6360679864883423,
+      "logits/chosen": -24.384828567504883,
+      "logits/rejected": -24.368749618530273,
+      "logps/chosen": -11.551971435546875,
+      "logps/rejected": -11.780753135681152,
+      "loss": 14.478,
+      "nll_loss": 14.314355850219727,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.1551971435546875,
+      "rewards/margins": 0.02287815511226654,
+      "rewards/rejected": -1.1780753135681152,
+      "step": 9
+    },
+    {
+      "epoch": 0.019704433497536946,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -4.276574611663818,
+      "log_odds_ratio": -4.645117282867432,
+      "logits/chosen": -24.044593811035156,
+      "logits/rejected": -24.136451721191406,
+      "logps/chosen": -14.244277000427246,
+      "logps/rejected": -9.967912673950195,
+      "loss": 14.9727,
+      "nll_loss": 14.508198738098145,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4244277477264404,
+      "rewards/margins": -0.4276365041732788,
+      "rewards/rejected": -0.9967912435531616,
+      "step": 10
+    },
+    {
+      "epoch": 0.02167487684729064,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -0.47632652521133423,
+      "log_odds_ratio": -2.1015360355377197,
+      "logits/chosen": -24.50082015991211,
+      "logits/rejected": -24.602977752685547,
+      "logps/chosen": -12.823345184326172,
+      "logps/rejected": -12.347034454345703,
+      "loss": 14.5178,
+      "nll_loss": 14.307616233825684,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.282334566116333,
+      "rewards/margins": -0.04763120412826538,
+      "rewards/rejected": -1.2347034215927124,
+      "step": 11
+    },
+    {
+      "epoch": 0.023645320197044337,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": 0.5796804428100586,
+      "log_odds_ratio": -1.477919101715088,
+      "logits/chosen": -24.575241088867188,
+      "logits/rejected": -24.883501052856445,
+      "logps/chosen": -13.402945518493652,
+      "logps/rejected": -13.98263168334961,
+      "loss": 14.7192,
+      "nll_loss": 14.571435928344727,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.340294599533081,
+      "rewards/margins": 0.0579686313867569,
+      "rewards/rejected": -1.3982632160186768,
+      "step": 12
+    },
+    {
+      "epoch": 0.02561576354679803,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -5.415220260620117,
+      "log_odds_ratio": -5.430206775665283,
+      "logits/chosen": -24.317058563232422,
+      "logits/rejected": -24.5443058013916,
+      "logps/chosen": -15.859424591064453,
+      "logps/rejected": -10.444257736206055,
+      "loss": 15.1246,
+      "nll_loss": 14.581562995910645,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -1.5859423875808716,
+      "rewards/margins": -0.5415166020393372,
+      "rewards/rejected": -1.0444257259368896,
+      "step": 13
+    },
+    {
+      "epoch": 0.027586206896551724,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "log_odds_chosen": 0.3306593894958496,
+      "log_odds_ratio": -1.4523016214370728,
+      "logits/chosen": -24.09428596496582,
+      "logits/rejected": -24.28605079650879,
+      "logps/chosen": -14.285757064819336,
+      "logps/rejected": -14.616400718688965,
+      "loss": 14.6036,
+      "nll_loss": 14.458335876464844,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4285757541656494,
+      "rewards/margins": 0.033064305782318115,
+      "rewards/rejected": -1.4616400003433228,
+      "step": 14
+    },
+    {
+      "epoch": 0.029556650246305417,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -1.3689701557159424,
+      "log_odds_ratio": -1.7255356311798096,
+      "logits/chosen": -25.194055557250977,
+      "logits/rejected": -25.31273651123047,
+      "logps/chosen": -14.159196853637695,
+      "logps/rejected": -12.790271759033203,
+      "loss": 14.5333,
+      "nll_loss": 14.360706329345703,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.4159196615219116,
+      "rewards/margins": -0.13689251244068146,
+      "rewards/rejected": -1.2790271043777466,
+      "step": 15
+    },
+    {
+      "epoch": 0.03152709359605911,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -2.09586501121521,
+      "log_odds_ratio": -2.7668588161468506,
+      "logits/chosen": -24.80086326599121,
+      "logits/rejected": -24.894508361816406,
+      "logps/chosen": -13.926980972290039,
+      "logps/rejected": -11.831639289855957,
+      "loss": 14.2473,
+      "nll_loss": 13.970612525939941,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -1.392698049545288,
+      "rewards/margins": -0.20953413844108582,
+      "rewards/rejected": -1.1831640005111694,
+      "step": 16
+    },
+    {
+      "epoch": 0.033497536945812804,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -1.7658007144927979,
+      "log_odds_ratio": -2.32814884185791,
+      "logits/chosen": -24.699962615966797,
+      "logits/rejected": -25.009201049804688,
+      "logps/chosen": -12.921096801757812,
+      "logps/rejected": -11.155325889587402,
+      "loss": 14.6062,
+      "nll_loss": 14.373355865478516,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.292109727859497,
+      "rewards/margins": -0.17657716572284698,
+      "rewards/rejected": -1.115532636642456,
+      "step": 17
+    },
+    {
+      "epoch": 0.035467980295566505,
+      "grad_norm": 6173175.5,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -0.547370195388794,
+      "log_odds_ratio": -2.4605464935302734,
+      "logits/chosen": -23.92157554626465,
+      "logits/rejected": -24.109575271606445,
+      "logps/chosen": -14.743122100830078,
+      "logps/rejected": -14.195760726928711,
+      "loss": 14.8543,
+      "nll_loss": 14.608250617980957,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.47431218624115,
+      "rewards/margins": -0.05473603308200836,
+      "rewards/rejected": -1.4195760488510132,
+      "step": 18
+    },
+    {
+      "epoch": 0.0374384236453202,
+      "grad_norm": 6015911.5,
+      "learning_rate": 2e-05,
+      "log_odds_chosen": 0.23701810836791992,
+      "log_odds_ratio": -1.323504090309143,
+      "logits/chosen": -23.915435791015625,
+      "logits/rejected": -24.436481475830078,
+      "logps/chosen": -13.316993713378906,
+      "logps/rejected": -13.554025650024414,
+      "loss": 14.4296,
+      "nll_loss": 14.297289848327637,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3316993713378906,
+      "rewards/margins": 0.023703187704086304,
+      "rewards/rejected": -1.3554025888442993,
+      "step": 19
+    },
+    {
+      "epoch": 0.03940886699507389,
+      "grad_norm": 5207.06689453125,
+      "learning_rate": 4e-05,
+      "log_odds_chosen": -1.192253589630127,
+      "log_odds_ratio": -1.5732345581054688,
+      "logits/chosen": -21.976619720458984,
+      "logits/rejected": -21.99953269958496,
+      "logps/chosen": -6.056412220001221,
+      "logps/rejected": -4.870516777038574,
+      "loss": 5.4013,
+      "nll_loss": 5.2440185546875,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.60564124584198,
+      "rewards/margins": -0.11858953535556793,
+      "rewards/rejected": -0.48705166578292847,
+      "step": 20
+    },
+    {
+      "epoch": 0.041379310344827586,
+      "grad_norm": 2327.033935546875,
+      "learning_rate": 6e-05,
+      "log_odds_chosen": -1.551367998123169,
+      "log_odds_ratio": -1.7944495677947998,
+      "logits/chosen": 0.21155580878257751,
+      "logits/rejected": 0.20273001492023468,
+      "logps/chosen": -6.774656295776367,
+      "logps/rejected": -5.233148574829102,
+      "loss": 7.3592,
+      "nll_loss": 7.179717540740967,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.6774656176567078,
+      "rewards/margins": -0.15415072441101074,
+      "rewards/rejected": -0.523314893245697,
+      "step": 21
+    },
+    {
+      "epoch": 0.04334975369458128,
+      "grad_norm": 310.5857849121094,
+      "learning_rate": 8e-05,
+      "log_odds_chosen": -0.1431032419204712,
+      "log_odds_ratio": -0.9799605011940002,
+      "logits/chosen": -3.0543479919433594,
+      "logits/rejected": -3.134338855743408,
+      "logps/chosen": -5.293187141418457,
+      "logps/rejected": -5.154700756072998,
+      "loss": 4.1911,
+      "nll_loss": 4.093069076538086,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5293187499046326,
+      "rewards/margins": -0.013848677277565002,
+      "rewards/rejected": -0.5154700875282288,
+      "step": 22
+    },
+    {
+      "epoch": 0.04532019704433497,
+      "grad_norm": 777.7794799804688,
+      "learning_rate": 0.0001,
+      "log_odds_chosen": -0.6350992918014526,
+      "log_odds_ratio": -1.130600929260254,
+      "logits/chosen": 0.6047357320785522,
+      "logits/rejected": 0.5388748049736023,
+      "logps/chosen": -5.000043869018555,
+      "logps/rejected": -4.370396614074707,
+      "loss": 3.7691,
+      "nll_loss": 3.656087875366211,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5000044107437134,
+      "rewards/margins": -0.0629647746682167,
+      "rewards/rejected": -0.43703967332839966,
+      "step": 23
+    },
+    {
+      "epoch": 0.04729064039408867,
+      "grad_norm": 283.93194580078125,
+      "learning_rate": 9.996713769306606e-05,
+      "log_odds_chosen": -2.414496660232544,
+      "log_odds_ratio": -2.642920732498169,
+      "logits/chosen": 2.4128100872039795,
+      "logits/rejected": 2.3816237449645996,
+      "logps/chosen": -7.231414318084717,
+      "logps/rejected": -4.824892997741699,
+      "loss": 3.8705,
+      "nll_loss": 3.606210947036743,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.7231414318084717,
+      "rewards/margins": -0.24065205454826355,
+      "rewards/rejected": -0.48248934745788574,
+      "step": 24
+    },
+    {
+      "epoch": 0.04926108374384237,
+      "grad_norm": 188.28753662109375,
+      "learning_rate": 9.993427538613212e-05,
+      "log_odds_chosen": -0.034127891063690186,
+      "log_odds_ratio": -1.0303078889846802,
+      "logits/chosen": 2.203524589538574,
+      "logits/rejected": 2.1993777751922607,
+      "logps/chosen": -4.807027816772461,
+      "logps/rejected": -4.762270927429199,
+      "loss": 2.7061,
+      "nll_loss": 2.6030540466308594,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.48070281744003296,
+      "rewards/margins": -0.004475705325603485,
+      "rewards/rejected": -0.4762271046638489,
+      "step": 25
+    },
+    {
+      "epoch": 0.05123152709359606,
+      "grad_norm": 102.36943817138672,
+      "learning_rate": 9.990141307919817e-05,
+      "log_odds_chosen": -1.2003881931304932,
+      "log_odds_ratio": -1.4746370315551758,
+      "logits/chosen": 2.1239614486694336,
+      "logits/rejected": 2.033912181854248,
+      "logps/chosen": -5.510244369506836,
+      "logps/rejected": -4.319775581359863,
+      "loss": 2.7212,
+      "nll_loss": 2.573741912841797,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.5510244369506836,
+      "rewards/margins": -0.1190468966960907,
+      "rewards/rejected": -0.43197759985923767,
+      "step": 26
+    },
+    {
+      "epoch": 0.053201970443349754,
+      "grad_norm": 81.96041870117188,
+      "learning_rate": 9.986855077226421e-05,
+      "log_odds_chosen": -0.9109505414962769,
+      "log_odds_ratio": -1.302980899810791,
+      "logits/chosen": 3.5120558738708496,
+      "logits/rejected": 3.4509434700012207,
+      "logps/chosen": -5.698228359222412,
+      "logps/rejected": -4.793853759765625,
+      "loss": 1.9828,
+      "nll_loss": 1.8524658679962158,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5698227882385254,
+      "rewards/margins": -0.0904374048113823,
+      "rewards/rejected": -0.4793854057788849,
+      "step": 27
+    },
+    {
+      "epoch": 0.05517241379310345,
+      "grad_norm": 84.83849334716797,
+      "learning_rate": 9.983568846533027e-05,
+      "log_odds_chosen": -1.7462624311447144,
+      "log_odds_ratio": -1.9260845184326172,
+      "logits/chosen": 3.9008326530456543,
+      "logits/rejected": 3.799042224884033,
+      "logps/chosen": -5.856660842895508,
+      "logps/rejected": -4.131113052368164,
+      "loss": 1.6424,
+      "nll_loss": 1.4497828483581543,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.5856660604476929,
+      "rewards/margins": -0.17255476117134094,
+      "rewards/rejected": -0.41311126947402954,
+      "step": 28
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "grad_norm": 88.78253173828125,
+      "learning_rate": 9.980282615839632e-05,
+      "log_odds_chosen": -0.9374311566352844,
+      "log_odds_ratio": -1.3175901174545288,
+      "logits/chosen": 3.8419084548950195,
+      "logits/rejected": 3.777817726135254,
+      "logps/chosen": -6.328927993774414,
+      "logps/rejected": -5.394232749938965,
+      "loss": 1.2995,
+      "nll_loss": 1.1677031517028809,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.6328928470611572,
+      "rewards/margins": -0.09346956014633179,
+      "rewards/rejected": -0.5394232869148254,
+      "step": 29
+    },
+    {
+      "epoch": 0.059113300492610835,
+      "grad_norm": 85.79216003417969,
+      "learning_rate": 9.976996385146237e-05,
+      "log_odds_chosen": 0.1410401165485382,
+      "log_odds_ratio": -0.7292813062667847,
+      "logits/chosen": 3.381721019744873,
+      "logits/rejected": 3.384875535964966,
+      "logps/chosen": -5.744163990020752,
+      "logps/rejected": -5.887864589691162,
+      "loss": 0.9669,
+      "nll_loss": 0.8939460515975952,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5744163990020752,
+      "rewards/margins": 0.014370076358318329,
+      "rewards/rejected": -0.5887864828109741,
+      "step": 30
+    },
+    {
+      "epoch": 0.061083743842364535,
+      "grad_norm": 107.95655822753906,
+      "learning_rate": 9.973710154452844e-05,
+      "log_odds_chosen": -0.25018808245658875,
+      "log_odds_ratio": -1.0048614740371704,
+      "logits/chosen": 3.1517415046691895,
+      "logits/rejected": 3.1261696815490723,
+      "logps/chosen": -5.209569454193115,
+      "logps/rejected": -4.958795070648193,
+      "loss": 1.0528,
+      "nll_loss": 0.9523099064826965,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5209569931030273,
+      "rewards/margins": -0.0250774547457695,
+      "rewards/rejected": -0.49587953090667725,
+      "step": 31
+    },
+    {
+      "epoch": 0.06305418719211822,
+      "grad_norm": 44.29023361206055,
+      "learning_rate": 9.970423923759449e-05,
+      "log_odds_chosen": -1.5885533094406128,
+      "log_odds_ratio": -1.8175604343414307,
+      "logits/chosen": 3.362605571746826,
+      "logits/rejected": 3.2463502883911133,
+      "logps/chosen": -6.499927520751953,
+      "logps/rejected": -4.917254447937012,
+      "loss": 1.0785,
+      "nll_loss": 0.8967477679252625,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.649992823600769,
+      "rewards/margins": -0.1582673192024231,
+      "rewards/rejected": -0.49172544479370117,
+      "step": 32
+    },
+    {
+      "epoch": 0.06502463054187192,
+      "grad_norm": 28.85852813720703,
+      "learning_rate": 9.967137693066053e-05,
+      "log_odds_chosen": -0.49852561950683594,
+      "log_odds_ratio": -1.0055984258651733,
+      "logits/chosen": 3.460742235183716,
+      "logits/rejected": 3.330906867980957,
+      "logps/chosen": -5.265803337097168,
+      "logps/rejected": -4.7726335525512695,
+      "loss": 1.0102,
+      "nll_loss": 0.9096041917800903,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.5265803933143616,
+      "rewards/margins": -0.04931700974702835,
+      "rewards/rejected": -0.47726333141326904,
+      "step": 33
+    },
+    {
+      "epoch": 0.06699507389162561,
+      "grad_norm": 22.544160842895508,
+      "learning_rate": 9.963851462372659e-05,
+      "log_odds_chosen": 0.10691678524017334,
+      "log_odds_ratio": -0.7760136723518372,
+      "logits/chosen": 3.7231876850128174,
+      "logits/rejected": 3.7375643253326416,
+      "logps/chosen": -5.739330768585205,
+      "logps/rejected": -5.8460869789123535,
+      "loss": 0.593,
+      "nll_loss": 0.5153771638870239,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5739331245422363,
+      "rewards/margins": 0.01067563146352768,
+      "rewards/rejected": -0.5846087336540222,
+      "step": 34
+    },
+    {
+      "epoch": 0.06896551724137931,
+      "grad_norm": 19.277795791625977,
+      "learning_rate": 9.960565231679264e-05,
+      "log_odds_chosen": 0.10966051369905472,
+      "log_odds_ratio": -0.6413217186927795,
+      "logits/chosen": 4.203195095062256,
+      "logits/rejected": 4.189924716949463,
+      "logps/chosen": -5.146385669708252,
+      "logps/rejected": -5.255435943603516,
+      "loss": 0.5091,
+      "nll_loss": 0.4449358880519867,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5146385431289673,
+      "rewards/margins": 0.010905012488365173,
+      "rewards/rejected": -0.5255435705184937,
+      "step": 35
+    },
+    {
+      "epoch": 0.07093596059113301,
+      "grad_norm": 28.63817596435547,
+      "learning_rate": 9.95727900098587e-05,
+      "log_odds_chosen": -0.6588072776794434,
+      "log_odds_ratio": -1.4066166877746582,
+      "logits/chosen": 4.6068220138549805,
+      "logits/rejected": 4.534092426300049,
+      "logps/chosen": -5.351911544799805,
+      "logps/rejected": -4.696127891540527,
+      "loss": 0.8579,
+      "nll_loss": 0.7172427177429199,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5351911187171936,
+      "rewards/margins": -0.06557835638523102,
+      "rewards/rejected": -0.4696127772331238,
+      "step": 36
+    },
+    {
+      "epoch": 0.0729064039408867,
+      "grad_norm": 16.950454711914062,
+      "learning_rate": 9.953992770292476e-05,
+      "log_odds_chosen": -0.26769834756851196,
+      "log_odds_ratio": -0.9218679666519165,
+      "logits/chosen": 4.716145038604736,
+      "logits/rejected": 4.708680629730225,
+      "logps/chosen": -4.707310676574707,
+      "logps/rejected": -4.44110107421875,
+      "loss": 0.4986,
+      "nll_loss": 0.40645352005958557,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.47073113918304443,
+      "rewards/margins": -0.026621006429195404,
+      "rewards/rejected": -0.44411009550094604,
+      "step": 37
+    },
+    {
+      "epoch": 0.0748768472906404,
+      "grad_norm": 24.627229690551758,
+      "learning_rate": 9.950706539599081e-05,
+      "log_odds_chosen": 0.25464457273483276,
+      "log_odds_ratio": -0.6431488394737244,
+      "logits/chosen": 5.137335777282715,
+      "logits/rejected": 4.8371148109436035,
+      "logps/chosen": -4.458625316619873,
+      "logps/rejected": -4.715159893035889,
+      "loss": 1.205,
+      "nll_loss": 1.1407045125961304,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4458625614643097,
+      "rewards/margins": 0.025653444230556488,
+      "rewards/rejected": -0.4715160131454468,
+      "step": 38
+    },
+    {
+      "epoch": 0.07684729064039408,
+      "grad_norm": 15.05199146270752,
+      "learning_rate": 9.947420308905686e-05,
+      "log_odds_chosen": 0.06547074019908905,
+      "log_odds_ratio": -0.7938849925994873,
+      "logits/chosen": 5.672910690307617,
+      "logits/rejected": 5.668852806091309,
+      "logps/chosen": -5.0126471519470215,
+      "logps/rejected": -5.076702117919922,
+      "loss": 0.6147,
+      "nll_loss": 0.5352977514266968,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.501264750957489,
+      "rewards/margins": 0.006405487656593323,
+      "rewards/rejected": -0.5076702833175659,
+      "step": 39
+    },
+    {
+      "epoch": 0.07881773399014778,
+      "grad_norm": 14.93382740020752,
+      "learning_rate": 9.944134078212291e-05,
+      "log_odds_chosen": -0.9602937698364258,
+      "log_odds_ratio": -1.3585858345031738,
+      "logits/chosen": 6.097533226013184,
+      "logits/rejected": 5.998549461364746,
+      "logps/chosen": -4.690290451049805,
+      "logps/rejected": -3.7499120235443115,
+      "loss": 0.7308,
+      "nll_loss": 0.5949816703796387,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.469029039144516,
+      "rewards/margins": -0.09403782337903976,
+      "rewards/rejected": -0.37499120831489563,
+      "step": 40
+    },
+    {
+      "epoch": 0.08078817733990148,
+      "grad_norm": 13.022683143615723,
+      "learning_rate": 9.940847847518896e-05,
+      "log_odds_chosen": -0.7364238500595093,
+      "log_odds_ratio": -1.2723374366760254,
+      "logits/chosen": 6.448332786560059,
+      "logits/rejected": 6.362303256988525,
+      "logps/chosen": -5.171870708465576,
+      "logps/rejected": -4.441474437713623,
+      "loss": 0.767,
+      "nll_loss": 0.6397515535354614,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5171870589256287,
+      "rewards/margins": -0.07303954660892487,
+      "rewards/rejected": -0.4441474676132202,
+      "step": 41
+    },
+    {
+      "epoch": 0.08275862068965517,
+      "grad_norm": 14.884285926818848,
+      "learning_rate": 9.937561616825501e-05,
+      "log_odds_chosen": -1.1798081398010254,
+      "log_odds_ratio": -1.5341217517852783,
+      "logits/chosen": 6.504058837890625,
+      "logits/rejected": 6.340629577636719,
+      "logps/chosen": -6.422999858856201,
+      "logps/rejected": -5.247575759887695,
+      "loss": 0.8761,
+      "nll_loss": 0.7226958870887756,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.642300009727478,
+      "rewards/margins": -0.11754237860441208,
+      "rewards/rejected": -0.5247576236724854,
+      "step": 42
+    },
+    {
+      "epoch": 0.08472906403940887,
+      "grad_norm": 13.257393836975098,
+      "learning_rate": 9.934275386132106e-05,
+      "log_odds_chosen": 0.12551864981651306,
+      "log_odds_ratio": -0.6492000818252563,
+      "logits/chosen": 6.410210609436035,
+      "logits/rejected": 6.107417106628418,
+      "logps/chosen": -4.046733379364014,
+      "logps/rejected": -4.169031620025635,
+      "loss": 0.7091,
+      "nll_loss": 0.64417564868927,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.40467336773872375,
+      "rewards/margins": 0.012229815125465393,
+      "rewards/rejected": -0.41690316796302795,
+      "step": 43
+    },
+    {
+      "epoch": 0.08669950738916256,
+      "grad_norm": 12.279217720031738,
+      "learning_rate": 9.930989155438713e-05,
+      "log_odds_chosen": -0.5023277997970581,
+      "log_odds_ratio": -1.0301179885864258,
+      "logits/chosen": 6.970707416534424,
+      "logits/rejected": 6.647186279296875,
+      "logps/chosen": -4.714241027832031,
+      "logps/rejected": -4.2168684005737305,
+      "loss": 0.6444,
+      "nll_loss": 0.5413457155227661,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4714241325855255,
+      "rewards/margins": -0.04973728209733963,
+      "rewards/rejected": -0.4216868281364441,
+      "step": 44
+    },
+    {
+      "epoch": 0.08866995073891626,
+      "grad_norm": 13.653495788574219,
+      "learning_rate": 9.927702924745318e-05,
+      "log_odds_chosen": -0.46324121952056885,
+      "log_odds_ratio": -1.038538932800293,
+      "logits/chosen": 7.254502773284912,
+      "logits/rejected": 7.006526470184326,
+      "logps/chosen": -5.2505412101745605,
+      "logps/rejected": -4.788273811340332,
+      "loss": 0.9008,
+      "nll_loss": 0.796896755695343,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5250540971755981,
+      "rewards/margins": -0.04622676968574524,
+      "rewards/rejected": -0.4788273572921753,
+      "step": 45
+    },
+    {
+      "epoch": 0.09064039408866995,
+      "grad_norm": 18.146709442138672,
+      "learning_rate": 9.924416694051923e-05,
+      "log_odds_chosen": -0.7738027572631836,
+      "log_odds_ratio": -1.176024317741394,
+      "logits/chosen": 8.201786041259766,
+      "logits/rejected": 7.724084854125977,
+      "logps/chosen": -4.732602119445801,
+      "logps/rejected": -3.9752197265625,
+      "loss": 1.0407,
+      "nll_loss": 0.923093318939209,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.47326022386550903,
+      "rewards/margins": -0.07573825865983963,
+      "rewards/rejected": -0.3975220024585724,
+      "step": 46
+    },
+    {
+      "epoch": 0.09261083743842365,
+      "grad_norm": 14.329615592956543,
+      "learning_rate": 9.921130463358528e-05,
+      "log_odds_chosen": -0.639348566532135,
+      "log_odds_ratio": -1.1404924392700195,
+      "logits/chosen": 8.942170143127441,
+      "logits/rejected": 8.155258178710938,
+      "logps/chosen": -4.639939308166504,
+      "logps/rejected": -4.001932144165039,
+      "loss": 0.8478,
+      "nll_loss": 0.7337337732315063,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.46399396657943726,
+      "rewards/margins": -0.06380070745944977,
+      "rewards/rejected": -0.4001932740211487,
+      "step": 47
+    },
+    {
+      "epoch": 0.09458128078817735,
+      "grad_norm": 17.466739654541016,
+      "learning_rate": 9.917844232665133e-05,
+      "log_odds_chosen": -0.25936323404312134,
+      "log_odds_ratio": -0.8763980269432068,
+      "logits/chosen": 9.03748893737793,
+      "logits/rejected": 8.71670913696289,
+      "logps/chosen": -6.151959419250488,
+      "logps/rejected": -5.893428325653076,
+      "loss": 0.5265,
+      "nll_loss": 0.4389019012451172,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6151959896087646,
+      "rewards/margins": -0.025853142142295837,
+      "rewards/rejected": -0.5893428325653076,
+      "step": 48
+    },
+    {
+      "epoch": 0.09655172413793103,
+      "grad_norm": 24.616870880126953,
+      "learning_rate": 9.914558001971738e-05,
+      "log_odds_chosen": 0.21263563632965088,
+      "log_odds_ratio": -0.7889174818992615,
+      "logits/chosen": 8.891792297363281,
+      "logits/rejected": 8.628280639648438,
+      "logps/chosen": -5.473745822906494,
+      "logps/rejected": -5.688695907592773,
+      "loss": 0.5961,
+      "nll_loss": 0.5172120332717896,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5473745465278625,
+      "rewards/margins": 0.021495021879673004,
+      "rewards/rejected": -0.5688695907592773,
+      "step": 49
+    },
+    {
+      "epoch": 0.09852216748768473,
+      "grad_norm": 14.071308135986328,
+      "learning_rate": 9.911271771278345e-05,
+      "log_odds_chosen": 0.504749596118927,
+      "log_odds_ratio": -0.6755421757698059,
+      "logits/chosen": 9.868827819824219,
+      "logits/rejected": 9.528295516967773,
+      "logps/chosen": -5.136891841888428,
+      "logps/rejected": -5.645229339599609,
+      "loss": 0.4809,
+      "nll_loss": 0.413296103477478,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5136892199516296,
+      "rewards/margins": 0.050833724439144135,
+      "rewards/rejected": -0.564522922039032,
+      "step": 50
+    },
+    {
+      "epoch": 0.10049261083743842,
+      "grad_norm": 14.168952941894531,
+      "learning_rate": 9.90798554058495e-05,
+      "log_odds_chosen": 1.6433926820755005,
+      "log_odds_ratio": -0.29421526193618774,
+      "logits/chosen": 10.09439754486084,
+      "logits/rejected": 9.682938575744629,
+      "logps/chosen": -5.242979049682617,
+      "logps/rejected": -6.872318267822266,
+      "loss": 0.4648,
+      "nll_loss": 0.43533527851104736,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5242978930473328,
+      "rewards/margins": 0.16293394565582275,
+      "rewards/rejected": -0.6872318387031555,
+      "step": 51
+    },
+    {
+      "epoch": 0.10246305418719212,
+      "grad_norm": 14.170472145080566,
+      "learning_rate": 9.904699309891555e-05,
+      "log_odds_chosen": 0.34438109397888184,
+      "log_odds_ratio": -0.7326459288597107,
+      "logits/chosen": 10.729270935058594,
+      "logits/rejected": 9.835092544555664,
+      "logps/chosen": -4.666653156280518,
+      "logps/rejected": -5.001763343811035,
+      "loss": 0.5973,
+      "nll_loss": 0.5240655541419983,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4666653275489807,
+      "rewards/margins": 0.03351101279258728,
+      "rewards/rejected": -0.5001763105392456,
+      "step": 52
+    },
+    {
+      "epoch": 0.10443349753694581,
+      "grad_norm": 17.02173614501953,
+      "learning_rate": 9.90141307919816e-05,
+      "log_odds_chosen": -0.38262391090393066,
+      "log_odds_ratio": -1.0096200704574585,
+      "logits/chosen": 10.967659950256348,
+      "logits/rejected": 10.65617561340332,
+      "logps/chosen": -5.771301746368408,
+      "logps/rejected": -5.392180919647217,
+      "loss": 0.6425,
+      "nll_loss": 0.5415604710578918,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5771301984786987,
+      "rewards/margins": -0.03791210800409317,
+      "rewards/rejected": -0.5392181277275085,
+      "step": 53
+    },
+    {
+      "epoch": 0.10640394088669951,
+      "grad_norm": 23.43153953552246,
+      "learning_rate": 9.898126848504765e-05,
+      "log_odds_chosen": 0.1213546022772789,
+      "log_odds_ratio": -0.6841877698898315,
+      "logits/chosen": 10.922224044799805,
+      "logits/rejected": 9.08872127532959,
+      "logps/chosen": -5.820175647735596,
+      "logps/rejected": -5.938196659088135,
+      "loss": 0.922,
+      "nll_loss": 0.8536195755004883,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5820176005363464,
+      "rewards/margins": 0.0118020698428154,
+      "rewards/rejected": -0.5938196778297424,
+      "step": 54
+    },
+    {
+      "epoch": 0.10837438423645321,
+      "grad_norm": 22.510122299194336,
+      "learning_rate": 9.89484061781137e-05,
+      "log_odds_chosen": -0.16780684888362885,
+      "log_odds_ratio": -0.8235434889793396,
+      "logits/chosen": 12.277153015136719,
+      "logits/rejected": 11.420037269592285,
+      "logps/chosen": -6.007660865783691,
+      "logps/rejected": -5.839876174926758,
+      "loss": 0.6628,
+      "nll_loss": 0.5804653167724609,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.6007661819458008,
+      "rewards/margins": -0.016778498888015747,
+      "rewards/rejected": -0.5839875936508179,
+      "step": 55
+    },
+    {
+      "epoch": 0.1103448275862069,
+      "grad_norm": 12.079493522644043,
+      "learning_rate": 9.891554387117977e-05,
+      "log_odds_chosen": 0.5339440703392029,
+      "log_odds_ratio": -0.7557147741317749,
+      "logits/chosen": 12.83298110961914,
+      "logits/rejected": 10.649534225463867,
+      "logps/chosen": -5.67183780670166,
+      "logps/rejected": -6.203601837158203,
+      "loss": 0.8928,
+      "nll_loss": 0.8171919584274292,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.567183792591095,
+      "rewards/margins": 0.05317641794681549,
+      "rewards/rejected": -0.6203601956367493,
+      "step": 56
+    },
+    {
+      "epoch": 0.1123152709359606,
+      "grad_norm": 12.565900802612305,
+      "learning_rate": 9.888268156424582e-05,
+      "log_odds_chosen": 2.0049660205841064,
+      "log_odds_ratio": -0.46851933002471924,
+      "logits/chosen": 13.357433319091797,
+      "logits/rejected": 12.244264602661133,
+      "logps/chosen": -4.6370134353637695,
+      "logps/rejected": -6.632777214050293,
+      "loss": 0.4339,
+      "nll_loss": 0.38705962896347046,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4637013077735901,
+      "rewards/margins": 0.19957643747329712,
+      "rewards/rejected": -0.6632777452468872,
+      "step": 57
+    },
+    {
+      "epoch": 0.11428571428571428,
+      "grad_norm": 22.761791229248047,
+      "learning_rate": 9.884981925731187e-05,
+      "log_odds_chosen": 4.712594509124756,
+      "log_odds_ratio": -0.06629680842161179,
+      "logits/chosen": 13.256675720214844,
+      "logits/rejected": 10.43091869354248,
+      "logps/chosen": -6.766969203948975,
+      "logps/rejected": -11.478358268737793,
+      "loss": 0.9561,
+      "nll_loss": 0.9494645595550537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6766969561576843,
+      "rewards/margins": 0.4711388945579529,
+      "rewards/rejected": -1.1478357315063477,
+      "step": 58
+    },
+    {
+      "epoch": 0.11625615763546798,
+      "grad_norm": 19.949363708496094,
+      "learning_rate": 9.881695695037793e-05,
+      "log_odds_chosen": 6.496246337890625,
+      "log_odds_ratio": -0.0037864374462515116,
+      "logits/chosen": 15.012136459350586,
+      "logits/rejected": 7.278632164001465,
+      "logps/chosen": -6.7131805419921875,
+      "logps/rejected": -13.207061767578125,
+      "loss": 1.5044,
+      "nll_loss": 1.5040128231048584,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6713180541992188,
+      "rewards/margins": 0.6493881940841675,
+      "rewards/rejected": -1.3207062482833862,
+      "step": 59
+    },
+    {
+      "epoch": 0.11822660098522167,
+      "grad_norm": 16.179443359375,
+      "learning_rate": 9.878409464344397e-05,
+      "log_odds_chosen": 0.8906295299530029,
+      "log_odds_ratio": -0.5235388278961182,
+      "logits/chosen": 15.120689392089844,
+      "logits/rejected": 13.71633529663086,
+      "logps/chosen": -5.739372253417969,
+      "logps/rejected": -6.6298909187316895,
+      "loss": 0.7339,
+      "nll_loss": 0.6815049648284912,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5739372372627258,
+      "rewards/margins": 0.08905186504125595,
+      "rewards/rejected": -0.6629891395568848,
+      "step": 60
+    },
+    {
+      "epoch": 0.12019704433497537,
+      "grad_norm": 11.286359786987305,
+      "learning_rate": 9.875123233651002e-05,
+      "log_odds_chosen": 0.07031165063381195,
+      "log_odds_ratio": -0.7708790302276611,
+      "logits/chosen": 16.67514419555664,
+      "logits/rejected": 16.322460174560547,
+      "logps/chosen": -5.570927619934082,
+      "logps/rejected": -5.644192695617676,
+      "loss": 0.5769,
+      "nll_loss": 0.49984389543533325,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5570927858352661,
+      "rewards/margins": 0.007326461374759674,
+      "rewards/rejected": -0.5644192695617676,
+      "step": 61
+    },
+    {
+      "epoch": 0.12216748768472907,
+      "grad_norm": 22.134653091430664,
+      "learning_rate": 9.871837002957608e-05,
+      "log_odds_chosen": 1.5623975992202759,
+      "log_odds_ratio": -0.5218886137008667,
+      "logits/chosen": 16.926355361938477,
+      "logits/rejected": 14.960272789001465,
+      "logps/chosen": -4.799847602844238,
+      "logps/rejected": -6.362458229064941,
+      "loss": 1.029,
+      "nll_loss": 0.9768041372299194,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4799848198890686,
+      "rewards/margins": 0.15626102685928345,
+      "rewards/rejected": -0.636245846748352,
+      "step": 62
+    },
+    {
+      "epoch": 0.12413793103448276,
+      "grad_norm": 7.160065650939941,
+      "learning_rate": 9.868550772264214e-05,
+      "log_odds_chosen": 3.1212353706359863,
+      "log_odds_ratio": -0.11819203943014145,
+      "logits/chosen": 16.581613540649414,
+      "logits/rejected": 13.962539672851562,
+      "logps/chosen": -6.666845798492432,
+      "logps/rejected": -9.786575317382812,
+      "loss": 0.5787,
+      "nll_loss": 0.5669109225273132,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.666684627532959,
+      "rewards/margins": 0.31197309494018555,
+      "rewards/rejected": -0.9786577224731445,
+      "step": 63
+    },
+    {
+      "epoch": 0.12610837438423644,
+      "grad_norm": 12.619917869567871,
+      "learning_rate": 9.865264541570819e-05,
+      "log_odds_chosen": 2.7222530841827393,
+      "log_odds_ratio": -0.16750650107860565,
+      "logits/chosen": 15.768547058105469,
+      "logits/rejected": 14.996335983276367,
+      "logps/chosen": -6.992615222930908,
+      "logps/rejected": -9.713621139526367,
+      "loss": 0.5202,
+      "nll_loss": 0.5034125447273254,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6992615461349487,
+      "rewards/margins": 0.2721005976200104,
+      "rewards/rejected": -0.9713621735572815,
+      "step": 64
+    },
+    {
+      "epoch": 0.12807881773399016,
+      "grad_norm": 7.2716264724731445,
+      "learning_rate": 9.861978310877425e-05,
+      "log_odds_chosen": 1.4915066957473755,
+      "log_odds_ratio": -0.3234131336212158,
+      "logits/chosen": 17.015321731567383,
+      "logits/rejected": 13.940305709838867,
+      "logps/chosen": -5.3171563148498535,
+      "logps/rejected": -6.803923606872559,
+      "loss": 0.7122,
+      "nll_loss": 0.6798163056373596,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5317156314849854,
+      "rewards/margins": 0.14867670834064484,
+      "rewards/rejected": -0.6803923845291138,
+      "step": 65
+    },
+    {
+      "epoch": 0.13004926108374384,
+      "grad_norm": 11.72496223449707,
+      "learning_rate": 9.85869208018403e-05,
+      "log_odds_chosen": 1.5479227304458618,
+      "log_odds_ratio": -0.31622636318206787,
+      "logits/chosen": 17.73503303527832,
+      "logits/rejected": 14.991783142089844,
+      "logps/chosen": -5.972833633422852,
+      "logps/rejected": -7.51870584487915,
+      "loss": 0.6427,
+      "nll_loss": 0.6110478639602661,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5972833633422852,
+      "rewards/margins": 0.1545872688293457,
+      "rewards/rejected": -0.7518705725669861,
+      "step": 66
+    },
+    {
+      "epoch": 0.13201970443349753,
+      "grad_norm": 11.443265914916992,
+      "learning_rate": 9.855405849490634e-05,
+      "log_odds_chosen": -0.2689421772956848,
+      "log_odds_ratio": -0.9036697149276733,
+      "logits/chosen": 20.24970245361328,
+      "logits/rejected": 19.627458572387695,
+      "logps/chosen": -4.855075359344482,
+      "logps/rejected": -4.594576358795166,
+      "loss": 0.6392,
+      "nll_loss": 0.548798680305481,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4855075478553772,
+      "rewards/margins": -0.026049919426441193,
+      "rewards/rejected": -0.4594576358795166,
+      "step": 67
+    },
+    {
+      "epoch": 0.13399014778325122,
+      "grad_norm": 13.213132858276367,
+      "learning_rate": 9.85211961879724e-05,
+      "log_odds_chosen": -0.17688444256782532,
+      "log_odds_ratio": -0.9180728793144226,
+      "logits/chosen": 21.5200138092041,
+      "logits/rejected": 19.65924072265625,
+      "logps/chosen": -4.6597747802734375,
+      "logps/rejected": -4.482782363891602,
+      "loss": 0.5915,
+      "nll_loss": 0.4997202754020691,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4659774899482727,
+      "rewards/margins": -0.017699219286441803,
+      "rewards/rejected": -0.4482782483100891,
+      "step": 68
+    },
+    {
+      "epoch": 0.13596059113300493,
+      "grad_norm": 11.499565124511719,
+      "learning_rate": 9.848833388103845e-05,
+      "log_odds_chosen": -1.2865281105041504,
+      "log_odds_ratio": -1.7101082801818848,
+      "logits/chosen": 22.102703094482422,
+      "logits/rejected": 20.308788299560547,
+      "logps/chosen": -5.065018653869629,
+      "logps/rejected": -3.7976715564727783,
+      "loss": 0.758,
+      "nll_loss": 0.5869788527488708,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5065018534660339,
+      "rewards/margins": -0.12673470377922058,
+      "rewards/rejected": -0.37976717948913574,
+      "step": 69
+    },
+    {
+      "epoch": 0.13793103448275862,
+      "grad_norm": 11.51899242401123,
+      "learning_rate": 9.84554715741045e-05,
+      "log_odds_chosen": -0.9771645665168762,
+      "log_odds_ratio": -1.3545218706130981,
+      "logits/chosen": 22.744874954223633,
+      "logits/rejected": 21.691293716430664,
+      "logps/chosen": -5.24151086807251,
+      "logps/rejected": -4.273284435272217,
+      "loss": 0.6136,
+      "nll_loss": 0.4781510829925537,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.524151086807251,
+      "rewards/margins": -0.09682260453701019,
+      "rewards/rejected": -0.4273284673690796,
+      "step": 70
+    },
+    {
+      "epoch": 0.1399014778325123,
+      "grad_norm": 8.12386417388916,
+      "learning_rate": 9.842260926717056e-05,
+      "log_odds_chosen": 5.560305595397949,
+      "log_odds_ratio": -0.007505061104893684,
+      "logits/chosen": 22.703018188476562,
+      "logits/rejected": 17.591384887695312,
+      "logps/chosen": -4.401796817779541,
+      "logps/rejected": -9.947122573852539,
+      "loss": 0.7665,
+      "nll_loss": 0.7657051682472229,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.440179705619812,
+      "rewards/margins": 0.5545325875282288,
+      "rewards/rejected": -0.994712233543396,
+      "step": 71
+    },
+    {
+      "epoch": 0.14187192118226602,
+      "grad_norm": 8.638449668884277,
+      "learning_rate": 9.838974696023662e-05,
+      "log_odds_chosen": 3.2441861629486084,
+      "log_odds_ratio": -0.3544306457042694,
+      "logits/chosen": 22.605329513549805,
+      "logits/rejected": 21.118627548217773,
+      "logps/chosen": -6.654872894287109,
+      "logps/rejected": -9.896977424621582,
+      "loss": 0.4645,
+      "nll_loss": 0.42901375889778137,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6654873490333557,
+      "rewards/margins": 0.3242104649543762,
+      "rewards/rejected": -0.9896978139877319,
+      "step": 72
+    },
+    {
+      "epoch": 0.1438423645320197,
+      "grad_norm": 10.07016372680664,
+      "learning_rate": 9.835688465330266e-05,
+      "log_odds_chosen": 2.0025105476379395,
+      "log_odds_ratio": -0.22305379807949066,
+      "logits/chosen": 22.805526733398438,
+      "logits/rejected": 21.572620391845703,
+      "logps/chosen": -6.828444004058838,
+      "logps/rejected": -8.829482078552246,
+      "loss": 0.5262,
+      "nll_loss": 0.5038906931877136,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6828444004058838,
+      "rewards/margins": 0.2001037448644638,
+      "rewards/rejected": -0.8829481601715088,
+      "step": 73
+    },
+    {
+      "epoch": 0.1458128078817734,
+      "grad_norm": 14.818687438964844,
+      "learning_rate": 9.832402234636872e-05,
+      "log_odds_chosen": 2.7169299125671387,
+      "log_odds_ratio": -0.14701755344867706,
+      "logits/chosen": 22.76614761352539,
+      "logits/rejected": 19.976293563842773,
+      "logps/chosen": -5.468857765197754,
+      "logps/rejected": -8.179182052612305,
+      "loss": 0.8698,
+      "nll_loss": 0.8550534844398499,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5468858480453491,
+      "rewards/margins": 0.2710324227809906,
+      "rewards/rejected": -0.8179182410240173,
+      "step": 74
+    },
+    {
+      "epoch": 0.1477832512315271,
+      "grad_norm": 9.151193618774414,
+      "learning_rate": 9.829116003943477e-05,
+      "log_odds_chosen": 0.5578998327255249,
+      "log_odds_ratio": -0.570715606212616,
+      "logits/chosen": 26.05361557006836,
+      "logits/rejected": 23.909162521362305,
+      "logps/chosen": -6.121988296508789,
+      "logps/rejected": -6.679934978485107,
+      "loss": 0.5737,
+      "nll_loss": 0.5166354775428772,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6121988296508789,
+      "rewards/margins": 0.05579468607902527,
+      "rewards/rejected": -0.6679935455322266,
+      "step": 75
+    },
+    {
+      "epoch": 0.1497536945812808,
+      "grad_norm": 10.814386367797852,
+      "learning_rate": 9.825829773250082e-05,
+      "log_odds_chosen": 0.12924879789352417,
+      "log_odds_ratio": -0.7650930285453796,
+      "logits/chosen": 27.84402847290039,
+      "logits/rejected": 25.711360931396484,
+      "logps/chosen": -5.907541275024414,
+      "logps/rejected": -6.033694267272949,
+      "loss": 0.7706,
+      "nll_loss": 0.694129228591919,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5907541513442993,
+      "rewards/margins": 0.012615293264389038,
+      "rewards/rejected": -0.6033694744110107,
+      "step": 76
+    },
+    {
+      "epoch": 0.15172413793103448,
+      "grad_norm": 9.947409629821777,
+      "learning_rate": 9.822543542556687e-05,
+      "log_odds_chosen": 0.4837239384651184,
+      "log_odds_ratio": -0.6970803737640381,
+      "logits/chosen": 28.5462703704834,
+      "logits/rejected": 27.02786636352539,
+      "logps/chosen": -5.140244960784912,
+      "logps/rejected": -5.620762825012207,
+      "loss": 0.6616,
+      "nll_loss": 0.5918952226638794,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5140244960784912,
+      "rewards/margins": 0.048051804304122925,
+      "rewards/rejected": -0.5620762705802917,
+      "step": 77
+    },
+    {
+      "epoch": 0.15369458128078817,
+      "grad_norm": 6.910487651824951,
+      "learning_rate": 9.819257311863294e-05,
+      "log_odds_chosen": 0.7711626887321472,
+      "log_odds_ratio": -0.6080074906349182,
+      "logits/chosen": 31.223302841186523,
+      "logits/rejected": 30.153894424438477,
+      "logps/chosen": -5.687531471252441,
+      "logps/rejected": -6.460568428039551,
+      "loss": 0.7172,
+      "nll_loss": 0.6564466953277588,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5687531232833862,
+      "rewards/margins": 0.07730375975370407,
+      "rewards/rejected": -0.6460568904876709,
+      "step": 78
+    },
+    {
+      "epoch": 0.15566502463054188,
+      "grad_norm": 11.983855247497559,
+      "learning_rate": 9.815971081169899e-05,
+      "log_odds_chosen": 2.681151866912842,
+      "log_odds_ratio": -0.09658822417259216,
+      "logits/chosen": 32.32258224487305,
+      "logits/rejected": 27.915340423583984,
+      "logps/chosen": -5.5876030921936035,
+      "logps/rejected": -8.260780334472656,
+      "loss": 1.0529,
+      "nll_loss": 1.0432281494140625,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5587603449821472,
+      "rewards/margins": 0.26731765270233154,
+      "rewards/rejected": -0.8260780572891235,
+      "step": 79
+    },
+    {
+      "epoch": 0.15763546798029557,
+      "grad_norm": 10.444053649902344,
+      "learning_rate": 9.812684850476503e-05,
+      "log_odds_chosen": 0.8076292872428894,
+      "log_odds_ratio": -0.7855444550514221,
+      "logits/chosen": 33.401397705078125,
+      "logits/rejected": 28.73697280883789,
+      "logps/chosen": -5.7650465965271,
+      "logps/rejected": -6.5672807693481445,
+      "loss": 1.0387,
+      "nll_loss": 0.9601866006851196,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.576504647731781,
+      "rewards/margins": 0.0802234634757042,
+      "rewards/rejected": -0.6567280888557434,
+      "step": 80
+    },
+    {
+      "epoch": 0.15960591133004925,
+      "grad_norm": 8.213254928588867,
+      "learning_rate": 9.809398619783109e-05,
+      "log_odds_chosen": 2.2469544410705566,
+      "log_odds_ratio": -0.19246259331703186,
+      "logits/chosen": 32.93449020385742,
+      "logits/rejected": 30.33302116394043,
+      "logps/chosen": -6.000814437866211,
+      "logps/rejected": -8.243157386779785,
+      "loss": 0.7731,
+      "nll_loss": 0.7538505792617798,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6000814437866211,
+      "rewards/margins": 0.22423428297042847,
+      "rewards/rejected": -0.8243157267570496,
+      "step": 81
+    },
+    {
+      "epoch": 0.16157635467980297,
+      "grad_norm": 8.932541847229004,
+      "learning_rate": 9.806112389089714e-05,
+      "log_odds_chosen": 1.4511587619781494,
+      "log_odds_ratio": -0.31698089838027954,
+      "logits/chosen": 34.55916976928711,
+      "logits/rejected": 31.638761520385742,
+      "logps/chosen": -5.059939384460449,
+      "logps/rejected": -6.506154537200928,
+      "loss": 0.7791,
+      "nll_loss": 0.7473548650741577,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5059939622879028,
+      "rewards/margins": 0.14462152123451233,
+      "rewards/rejected": -0.6506154537200928,
+      "step": 82
+    },
+    {
+      "epoch": 0.16354679802955666,
+      "grad_norm": 10.539612770080566,
+      "learning_rate": 9.80282615839632e-05,
+      "log_odds_chosen": 5.860550880432129,
+      "log_odds_ratio": -0.051304906606674194,
+      "logits/chosen": 33.76152420043945,
+      "logits/rejected": 28.57237434387207,
+      "logps/chosen": -6.9805731773376465,
+      "logps/rejected": -12.839984893798828,
+      "loss": 1.2161,
+      "nll_loss": 1.2109378576278687,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6980573534965515,
+      "rewards/margins": 0.5859411954879761,
+      "rewards/rejected": -1.2839986085891724,
+      "step": 83
+    },
+    {
+      "epoch": 0.16551724137931034,
+      "grad_norm": 7.1307220458984375,
+      "learning_rate": 9.799539927702926e-05,
+      "log_odds_chosen": 2.0690383911132812,
+      "log_odds_ratio": -0.25591370463371277,
+      "logits/chosen": 33.75245666503906,
+      "logits/rejected": 32.03097915649414,
+      "logps/chosen": -4.031613349914551,
+      "logps/rejected": -6.090118408203125,
+      "loss": 0.7151,
+      "nll_loss": 0.6894702911376953,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40316134691238403,
+      "rewards/margins": 0.2058505415916443,
+      "rewards/rejected": -0.6090118885040283,
+      "step": 84
+    },
+    {
+      "epoch": 0.16748768472906403,
+      "grad_norm": 9.036500930786133,
+      "learning_rate": 9.796253697009531e-05,
+      "log_odds_chosen": 2.716524124145508,
+      "log_odds_ratio": -0.48059070110321045,
+      "logits/chosen": 33.204673767089844,
+      "logits/rejected": 32.63907241821289,
+      "logps/chosen": -6.555218696594238,
+      "logps/rejected": -9.271570205688477,
+      "loss": 0.3664,
+      "nll_loss": 0.31831711530685425,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6555218696594238,
+      "rewards/margins": 0.2716352343559265,
+      "rewards/rejected": -0.9271571040153503,
+      "step": 85
+    },
+    {
+      "epoch": 0.16945812807881774,
+      "grad_norm": 5.074675559997559,
+      "learning_rate": 9.792967466316136e-05,
+      "log_odds_chosen": 3.738826036453247,
+      "log_odds_ratio": -0.23348753154277802,
+      "logits/chosen": 33.88203430175781,
+      "logits/rejected": 32.66779327392578,
+      "logps/chosen": -5.057276725769043,
+      "logps/rejected": -8.792778015136719,
+      "loss": 0.3577,
+      "nll_loss": 0.3343806266784668,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5057277083396912,
+      "rewards/margins": 0.3735501170158386,
+      "rewards/rejected": -0.8792778253555298,
+      "step": 86
+    },
+    {
+      "epoch": 0.17142857142857143,
+      "grad_norm": 4.269649028778076,
+      "learning_rate": 9.789681235622741e-05,
+      "log_odds_chosen": 2.4954917430877686,
+      "log_odds_ratio": -0.13667386770248413,
+      "logits/chosen": 34.36510467529297,
+      "logits/rejected": 33.45208740234375,
+      "logps/chosen": -5.133493900299072,
+      "logps/rejected": -7.622622489929199,
+      "loss": 0.503,
+      "nll_loss": 0.4893127679824829,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5133493542671204,
+      "rewards/margins": 0.24891287088394165,
+      "rewards/rejected": -0.7622622847557068,
+      "step": 87
+    },
+    {
+      "epoch": 0.17339901477832512,
+      "grad_norm": 8.977710723876953,
+      "learning_rate": 9.786395004929346e-05,
+      "log_odds_chosen": 4.327927589416504,
+      "log_odds_ratio": -0.3331856429576874,
+      "logits/chosen": 34.582908630371094,
+      "logits/rejected": 30.748470306396484,
+      "logps/chosen": -8.300532341003418,
+      "logps/rejected": -12.627687454223633,
+      "loss": 1.211,
+      "nll_loss": 1.1777114868164062,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8300532698631287,
+      "rewards/margins": 0.432715505361557,
+      "rewards/rejected": -1.2627687454223633,
+      "step": 88
+    },
+    {
+      "epoch": 0.17536945812807883,
+      "grad_norm": 7.094234466552734,
+      "learning_rate": 9.783108774235951e-05,
+      "log_odds_chosen": 3.487314224243164,
+      "log_odds_ratio": -0.2728886604309082,
+      "logits/chosen": 35.85606384277344,
+      "logits/rejected": 31.37416648864746,
+      "logps/chosen": -4.788017272949219,
+      "logps/rejected": -8.273536682128906,
+      "loss": 1.0936,
+      "nll_loss": 1.0662983655929565,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.47880175709724426,
+      "rewards/margins": 0.3485518991947174,
+      "rewards/rejected": -0.8273537158966064,
+      "step": 89
+    },
+    {
+      "epoch": 0.17733990147783252,
+      "grad_norm": 6.6508378982543945,
+      "learning_rate": 9.779822543542558e-05,
+      "log_odds_chosen": 0.5686554908752441,
+      "log_odds_ratio": -0.4718419909477234,
+      "logits/chosen": 37.26837921142578,
+      "logits/rejected": 36.6246337890625,
+      "logps/chosen": -4.937107086181641,
+      "logps/rejected": -5.500895023345947,
+      "loss": 0.591,
+      "nll_loss": 0.543811023235321,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4937106966972351,
+      "rewards/margins": 0.056378819048404694,
+      "rewards/rejected": -0.5500895380973816,
+      "step": 90
+    },
+    {
+      "epoch": 0.1793103448275862,
+      "grad_norm": 8.963587760925293,
+      "learning_rate": 9.776536312849163e-05,
+      "log_odds_chosen": 0.17035016417503357,
+      "log_odds_ratio": -0.7936185598373413,
+      "logits/chosen": 38.87269592285156,
+      "logits/rejected": 37.28191375732422,
+      "logps/chosen": -5.533909797668457,
+      "logps/rejected": -5.704503059387207,
+      "loss": 0.8835,
+      "nll_loss": 0.8040995597839355,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5533909797668457,
+      "rewards/margins": 0.017059363424777985,
+      "rewards/rejected": -0.5704503059387207,
+      "step": 91
+    },
+    {
+      "epoch": 0.1812807881773399,
+      "grad_norm": 12.501840591430664,
+      "learning_rate": 9.773250082155768e-05,
+      "log_odds_chosen": 2.2529821395874023,
+      "log_odds_ratio": -0.6428291201591492,
+      "logits/chosen": 39.92913818359375,
+      "logits/rejected": 35.196495056152344,
+      "logps/chosen": -5.412675857543945,
+      "logps/rejected": -7.668416500091553,
+      "loss": 1.1262,
+      "nll_loss": 1.061893105506897,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5412676334381104,
+      "rewards/margins": 0.22557401657104492,
+      "rewards/rejected": -0.7668416500091553,
+      "step": 92
+    },
+    {
+      "epoch": 0.1832512315270936,
+      "grad_norm": 6.905440330505371,
+      "learning_rate": 9.769963851462373e-05,
+      "log_odds_chosen": 3.314648151397705,
+      "log_odds_ratio": -0.2885361313819885,
+      "logits/chosen": 42.369606018066406,
+      "logits/rejected": 37.715675354003906,
+      "logps/chosen": -4.620293140411377,
+      "logps/rejected": -7.927330493927002,
+      "loss": 0.7934,
+      "nll_loss": 0.7645502686500549,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4620293378829956,
+      "rewards/margins": 0.3307037949562073,
+      "rewards/rejected": -0.7927330732345581,
+      "step": 93
+    },
+    {
+      "epoch": 0.1852216748768473,
+      "grad_norm": 9.869806289672852,
+      "learning_rate": 9.766677620768978e-05,
+      "log_odds_chosen": -0.8111425042152405,
+      "log_odds_ratio": -2.0276551246643066,
+      "logits/chosen": 42.48541259765625,
+      "logits/rejected": 42.15509796142578,
+      "logps/chosen": -6.331485748291016,
+      "logps/rejected": -5.520636558532715,
+      "loss": 0.719,
+      "nll_loss": 0.5162821412086487,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6331485509872437,
+      "rewards/margins": -0.08108486235141754,
+      "rewards/rejected": -0.5520636439323425,
+      "step": 94
+    },
+    {
+      "epoch": 0.18719211822660098,
+      "grad_norm": 6.669962406158447,
+      "learning_rate": 9.763391390075583e-05,
+      "log_odds_chosen": 0.532020628452301,
+      "log_odds_ratio": -1.0998200178146362,
+      "logits/chosen": 43.793216705322266,
+      "logits/rejected": 42.82099151611328,
+      "logps/chosen": -4.958026885986328,
+      "logps/rejected": -5.452145576477051,
+      "loss": 0.7459,
+      "nll_loss": 0.6359626650810242,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.49580270051956177,
+      "rewards/margins": 0.04941190034151077,
+      "rewards/rejected": -0.5452145934104919,
+      "step": 95
+    },
+    {
+      "epoch": 0.1891625615763547,
+      "grad_norm": 6.615627765655518,
+      "learning_rate": 9.76010515938219e-05,
+      "log_odds_chosen": 2.370893955230713,
+      "log_odds_ratio": -0.3279552757740021,
+      "logits/chosen": 42.825660705566406,
+      "logits/rejected": 40.46846008300781,
+      "logps/chosen": -4.994439601898193,
+      "logps/rejected": -7.364171504974365,
+      "loss": 0.5388,
+      "nll_loss": 0.505994439125061,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.49944397807121277,
+      "rewards/margins": 0.23697316646575928,
+      "rewards/rejected": -0.7364171743392944,
+      "step": 96
+    },
+    {
+      "epoch": 0.19113300492610838,
+      "grad_norm": 4.877010822296143,
+      "learning_rate": 9.756818928688795e-05,
+      "log_odds_chosen": 1.899808406829834,
+      "log_odds_ratio": -0.4587392508983612,
+      "logits/chosen": 43.57826614379883,
+      "logits/rejected": 42.43614959716797,
+      "logps/chosen": -4.869477272033691,
+      "logps/rejected": -6.7610883712768555,
+      "loss": 0.5226,
+      "nll_loss": 0.4767411947250366,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4869477152824402,
+      "rewards/margins": 0.18916118144989014,
+      "rewards/rejected": -0.6761089563369751,
+      "step": 97
+    },
+    {
+      "epoch": 0.19310344827586207,
+      "grad_norm": 4.587172985076904,
+      "learning_rate": 9.7535326979954e-05,
+      "log_odds_chosen": 3.665987253189087,
+      "log_odds_ratio": -0.029790375381708145,
+      "logits/chosen": 43.945030212402344,
+      "logits/rejected": 40.424922943115234,
+      "logps/chosen": -3.78873610496521,
+      "logps/rejected": -7.397561550140381,
+      "loss": 0.4719,
+      "nll_loss": 0.4688977003097534,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3788736164569855,
+      "rewards/margins": 0.36088258028030396,
+      "rewards/rejected": -0.7397562265396118,
+      "step": 98
+    },
+    {
+      "epoch": 0.19507389162561575,
+      "grad_norm": 10.64942455291748,
+      "learning_rate": 9.750246467302005e-05,
+      "log_odds_chosen": -0.48347488045692444,
+      "log_odds_ratio": -1.076609492301941,
+      "logits/chosen": 44.01980972290039,
+      "logits/rejected": 43.58686828613281,
+      "logps/chosen": -5.610145568847656,
+      "logps/rejected": -5.123292446136475,
+      "loss": 0.5851,
+      "nll_loss": 0.477483332157135,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5610145330429077,
+      "rewards/margins": -0.04868524521589279,
+      "rewards/rejected": -0.5123292803764343,
+      "step": 99
+    },
+    {
+      "epoch": 0.19704433497536947,
+      "grad_norm": 5.142813682556152,
+      "learning_rate": 9.74696023660861e-05,
+      "log_odds_chosen": 2.760999917984009,
+      "log_odds_ratio": -0.07328092306852341,
+      "logits/chosen": 44.26462173461914,
+      "logits/rejected": 41.45793914794922,
+      "logps/chosen": -5.391923904418945,
+      "logps/rejected": -8.144350051879883,
+      "loss": 0.4225,
+      "nll_loss": 0.41515105962753296,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5391924381256104,
+      "rewards/margins": 0.27524256706237793,
+      "rewards/rejected": -0.8144350051879883,
+      "step": 100
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 3048,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c2bde389fa3e017892081623426bfac69048c5dde4b837375d0a543b0b8b500
+size 6161