Upload folder using huggingface_hub

adapter_config.json

@@ -25,12 +25,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "up_proj",
     "v_proj",
-    "k_proj",
     "gate_proj",
-    "q_proj",
+    "k_proj",
     "o_proj",
-    "up_proj",
+    "q_proj",
     "down_proj"
   ],
   "target_parameters": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd56087c89c66211706ec9604f874f7f37183adc4a87bde3296c02d95ad546cf
+oid sha256:8bad6519f4222c69631b39ae11e4e641593524e17b56b28f272222f981b37964
 size 70430032

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94d540d4d5ec8bd8a9745d5a6982fe34bd12584b54f1a20f08af3b91e1c1042e
+oid sha256:10f0af0adccf49d58dc86ff958136fdbc89448a8666cadb0cced82e40b534cd1
 size 36140325

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2b06adf858fd431254cfea4a735b5073ec95a938351985d2f58fd75afb88f51
+oid sha256:1cfde38fbb26ecac9770a5a0fdd06ad8687211a169ed61ebc2799cda44f36fb9
 size 14645

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_global_step": 100,
-  "best_metric": 0.3803686201572418,
-  "best_model_checkpoint": "./lora_out/1ca02918/checkpoint-100",
+  "best_metric": 0.3854297995567322,
+  "best_model_checkpoint": "./lora_out/efu0wyi4/checkpoint-100",
   "epoch": 1.0,
   "eval_steps": 10,
   "global_step": 107,
@@ -10,193 +10,213 @@
   "is_world_process_zero": true,
   "log_history": [
     {
+      "entropy": 1.1153396010398864,
       "epoch": 0.09411764705882353,
-      "grad_norm": 1.270073413848877,
+      "grad_norm": 1.2214514017105103,
       "learning_rate": 0.00016003636363636365,
-      "loss": 1.2398,
-      "mean_token_accuracy": 0.7614437624812126,
-      "num_tokens": 115124.0,
+      "loss": 1.2524,
+      "mean_token_accuracy": 0.7594971776008606,
+      "num_tokens": 115799.0,
       "step": 10
     },
     {
       "epoch": 0.09411764705882353,
-      "eval_loss": 0.896133542060852,
-      "eval_mean_token_accuracy": 0.8207377485434214,
-      "eval_num_tokens": 115124.0,
-      "eval_runtime": 23.5565,
-      "eval_samples_per_second": 6.368,
-      "eval_steps_per_second": 6.368,
+      "eval_entropy": 0.9907166957855225,
+      "eval_loss": 0.9035767912864685,
+      "eval_mean_token_accuracy": 0.8175230169296265,
+      "eval_num_tokens": 115799.0,
+      "eval_runtime": 26.0099,
+      "eval_samples_per_second": 5.767,
+      "eval_steps_per_second": 5.767,
       "step": 10
     },
     {
+      "entropy": 0.6309416361153126,
       "epoch": 0.18823529411764706,
-      "grad_norm": 0.6748596429824829,
+      "grad_norm": 0.6099486351013184,
       "learning_rate": 0.0001793,
-      "loss": 0.555,
-      "mean_token_accuracy": 0.8851501628756523,
-      "num_tokens": 183590.0,
+      "loss": 0.5626,
+      "mean_token_accuracy": 0.8832408726215363,
+      "num_tokens": 184486.0,
       "step": 20
     },
     {
       "epoch": 0.18823529411764706,
-      "eval_loss": 0.4925537109375,
-      "eval_mean_token_accuracy": 0.9008145872751872,
-      "eval_num_tokens": 183590.0,
-      "eval_runtime": 23.5261,
-      "eval_samples_per_second": 6.376,
-      "eval_steps_per_second": 6.376,
+      "eval_entropy": 0.4709713250398636,
+      "eval_loss": 0.5018168687820435,
+      "eval_mean_token_accuracy": 0.8997142084439596,
+      "eval_num_tokens": 184486.0,
+      "eval_runtime": 26.0226,
+      "eval_samples_per_second": 5.764,
+      "eval_steps_per_second": 5.764,
       "step": 20
     },
     {
+      "entropy": 0.5151705276221037,
       "epoch": 0.2823529411764706,
-      "grad_norm": 0.5109054446220398,
+      "grad_norm": 0.5180553793907166,
       "learning_rate": 0.000158925,
-      "loss": 0.5305,
-      "mean_token_accuracy": 0.8950182288885117,
-      "num_tokens": 274669.0,
+      "loss": 0.5382,
+      "mean_token_accuracy": 0.8940704673528671,
+      "num_tokens": 275845.0,
       "step": 30
     },
     {
       "epoch": 0.2823529411764706,
-      "eval_loss": 0.43484488129615784,
-      "eval_mean_token_accuracy": 0.9108006227016449,
-      "eval_num_tokens": 274669.0,
-      "eval_runtime": 23.5726,
-      "eval_samples_per_second": 6.363,
-      "eval_steps_per_second": 6.363,
+      "eval_entropy": 0.4590779893596967,
+      "eval_loss": 0.4454800486564636,
+      "eval_mean_token_accuracy": 0.9063887639840444,
+      "eval_num_tokens": 275845.0,
+      "eval_runtime": 26.0809,
+      "eval_samples_per_second": 5.751,
+      "eval_steps_per_second": 5.751,
       "step": 30
     },
     {
+      "entropy": 0.4594229131937027,
       "epoch": 0.3764705882352941,
-      "grad_norm": 0.2931932806968689,
+      "grad_norm": 0.2898012399673462,
       "learning_rate": 0.00013855,
-      "loss": 0.4322,
-      "mean_token_accuracy": 0.9151976436376572,
-      "num_tokens": 366523.0,
+      "loss": 0.45,
+      "mean_token_accuracy": 0.91114012748003,
+      "num_tokens": 368424.0,
       "step": 40
     },
     {
       "epoch": 0.3764705882352941,
-      "eval_loss": 0.419387549161911,
-      "eval_mean_token_accuracy": 0.9135305122534434,
-      "eval_num_tokens": 366523.0,
-      "eval_runtime": 23.518,
-      "eval_samples_per_second": 6.378,
-      "eval_steps_per_second": 6.378,
+      "eval_entropy": 0.4673765400548776,
+      "eval_loss": 0.43020305037498474,
+      "eval_mean_token_accuracy": 0.9115767550468444,
+      "eval_num_tokens": 368424.0,
+      "eval_runtime": 26.0231,
+      "eval_samples_per_second": 5.764,
+      "eval_steps_per_second": 5.764,
       "step": 40
     },
     {
+      "entropy": 0.3577877376228571,
       "epoch": 0.47058823529411764,
-      "grad_norm": 0.2669583559036255,
+      "grad_norm": 0.25978532433509827,
       "learning_rate": 0.000118175,
-      "loss": 0.3397,
-      "mean_token_accuracy": 0.9227273896336555,
-      "num_tokens": 427601.0,
+      "loss": 0.343,
+      "mean_token_accuracy": 0.9222506016492844,
+      "num_tokens": 429589.0,
       "step": 50
     },
     {
       "epoch": 0.47058823529411764,
-      "eval_loss": 0.4084239900112152,
-      "eval_mean_token_accuracy": 0.9155814254283905,
-      "eval_num_tokens": 427601.0,
-      "eval_runtime": 23.5117,
-      "eval_samples_per_second": 6.38,
-      "eval_steps_per_second": 6.38,
+      "eval_entropy": 0.39317929953336717,
+      "eval_loss": 0.42108139395713806,
+      "eval_mean_token_accuracy": 0.9134278730551402,
+      "eval_num_tokens": 429589.0,
+      "eval_runtime": 26.002,
+      "eval_samples_per_second": 5.769,
+      "eval_steps_per_second": 5.769,
       "step": 50
     },
     {
+      "entropy": 0.54765380397439,
       "epoch": 0.5647058823529412,
-      "grad_norm": 0.24561642110347748,
+      "grad_norm": 0.2597196698188782,
       "learning_rate": 9.78e-05,
-      "loss": 0.5525,
-      "mean_token_accuracy": 0.8890453979372979,
-      "num_tokens": 541710.0,
+      "loss": 0.5644,
+      "mean_token_accuracy": 0.8865802466869355,
+      "num_tokens": 544259.0,
       "step": 60
     },
     {
       "epoch": 0.5647058823529412,
-      "eval_loss": 0.4013407528400421,
-      "eval_mean_token_accuracy": 0.9171908668677012,
-      "eval_num_tokens": 541710.0,
-      "eval_runtime": 23.5555,
-      "eval_samples_per_second": 6.368,
-      "eval_steps_per_second": 6.368,
+      "eval_entropy": 0.41075391257802646,
+      "eval_loss": 0.4073421359062195,
+      "eval_mean_token_accuracy": 0.9149504574139913,
+      "eval_num_tokens": 544259.0,
+      "eval_runtime": 26.2517,
+      "eval_samples_per_second": 5.714,
+      "eval_steps_per_second": 5.714,
       "step": 60
     },
     {
+      "entropy": 0.3822147287428379,
       "epoch": 0.6588235294117647,
-      "grad_norm": 0.3256433606147766,
+      "grad_norm": 0.3055135905742645,
       "learning_rate": 7.7425e-05,
-      "loss": 0.3582,
-      "mean_token_accuracy": 0.9248881861567497,
-      "num_tokens": 620339.0,
+      "loss": 0.3693,
+      "mean_token_accuracy": 0.922209607064724,
+      "num_tokens": 623365.0,
       "step": 70
     },
     {
       "epoch": 0.6588235294117647,
-      "eval_loss": 0.3908792734146118,
-      "eval_mean_token_accuracy": 0.9177969670295716,
-      "eval_num_tokens": 620339.0,
-      "eval_runtime": 23.6388,
-      "eval_samples_per_second": 6.345,
-      "eval_steps_per_second": 6.345,
+      "eval_entropy": 0.4195191798110803,
+      "eval_loss": 0.3989087641239166,
+      "eval_mean_token_accuracy": 0.915905403693517,
+      "eval_num_tokens": 623365.0,
+      "eval_runtime": 26.2348,
+      "eval_samples_per_second": 5.718,
+      "eval_steps_per_second": 5.718,
       "step": 70
     },
     {
+      "entropy": 0.38976135551929475,
       "epoch": 0.7529411764705882,
-      "grad_norm": 0.35445523262023926,
+      "grad_norm": 0.34105750918388367,
       "learning_rate": 5.7050000000000004e-05,
-      "loss": 0.3729,
-      "mean_token_accuracy": 0.9206046536564827,
-      "num_tokens": 695812.0,
+      "loss": 0.3761,
+      "mean_token_accuracy": 0.9197401210665703,
+      "num_tokens": 699013.0,
       "step": 80
     },
     {
       "epoch": 0.7529411764705882,
-      "eval_loss": 0.3841544985771179,
-      "eval_mean_token_accuracy": 0.9190148003896077,
-      "eval_num_tokens": 695812.0,
-      "eval_runtime": 23.5427,
-      "eval_samples_per_second": 6.371,
-      "eval_steps_per_second": 6.371,
+      "eval_entropy": 0.39603232031067215,
+      "eval_loss": 0.3924122452735901,
+      "eval_mean_token_accuracy": 0.917570983171463,
+      "eval_num_tokens": 699013.0,
+      "eval_runtime": 26.1772,
+      "eval_samples_per_second": 5.73,
+      "eval_steps_per_second": 5.73,
       "step": 80
     },
     {
+      "entropy": 0.47345383167266847,
       "epoch": 0.8470588235294118,
-      "grad_norm": 0.2725517153739929,
+      "grad_norm": 0.3139539957046509,
       "learning_rate": 3.6675000000000004e-05,
-      "loss": 0.4558,
-      "mean_token_accuracy": 0.9098866626620292,
-      "num_tokens": 796104.0,
+      "loss": 0.4697,
+      "mean_token_accuracy": 0.9068517610430717,
+      "num_tokens": 800061.0,
       "step": 90
     },
     {
       "epoch": 0.8470588235294118,
-      "eval_loss": 0.3851577639579773,
-      "eval_mean_token_accuracy": 0.9192596852779389,
-      "eval_num_tokens": 796104.0,
-      "eval_runtime": 23.599,
-      "eval_samples_per_second": 6.356,
-      "eval_steps_per_second": 6.356,
+      "eval_entropy": 0.38997318550944327,
+      "eval_loss": 0.39237019419670105,
+      "eval_mean_token_accuracy": 0.917557996114095,
+      "eval_num_tokens": 800061.0,
+      "eval_runtime": 26.0084,
+      "eval_samples_per_second": 5.767,
+      "eval_steps_per_second": 5.767,
       "step": 90
     },
     {
+      "entropy": 0.2872362457215786,
       "epoch": 0.9411764705882353,
-      "grad_norm": 0.28482893109321594,
+      "grad_norm": 0.29094481468200684,
       "learning_rate": 1.63e-05,
-      "loss": 0.2851,
-      "mean_token_accuracy": 0.9351509854197502,
-      "num_tokens": 861517.0,
+      "loss": 0.2916,
+      "mean_token_accuracy": 0.9335257709026337,
+      "num_tokens": 865712.0,
       "step": 100
     },
     {
       "epoch": 0.9411764705882353,
-      "eval_loss": 0.3803686201572418,
-      "eval_mean_token_accuracy": 0.920326874256134,
-      "eval_num_tokens": 861517.0,
-      "eval_runtime": 23.6491,
-      "eval_samples_per_second": 6.343,
-      "eval_steps_per_second": 6.343,
+      "eval_entropy": 0.3926775233944257,
+      "eval_loss": 0.3854297995567322,
+      "eval_mean_token_accuracy": 0.9184372560183207,
+      "eval_num_tokens": 865712.0,
+      "eval_runtime": 26.027,
+      "eval_samples_per_second": 5.763,
+      "eval_steps_per_second": 5.763,
       "step": 100
     }
   ],
@@ -217,7 +237,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2032207927076352.0,
+  "total_flos": 2041922721504768.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c99c3d3d0bfedc2d4a25d224f204b007b771f3ffcf74f9edf57603f164cfe386
-size 6097
+oid sha256:4f775e201d96104912459fe7cf55fed26f17379938713adb7401917362d5db8d
+size 6161