Upload resid_post_layer_30/trainer_5/delta_lm_loss(unmask).json with huggingface_hub
Browse files
resid_post_layer_30/trainer_5/delta_lm_loss(unmask).json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokens_unmasked_evaluated": 2624920.0,
|
| 3 |
+
"dream_weighted_loss_clean(unmask)": 8.96004144888225,
|
| 4 |
+
"dream_weighted_loss_sae(unmask)": 8.672447160294409,
|
| 5 |
+
"delta_lm_loss(unmask)": -0.287594288587842,
|
| 6 |
+
"weighting": "unmask-only CE weighted by 1/(1 - t), first token excluded",
|
| 7 |
+
"t_min": 0.05,
|
| 8 |
+
"t_max": 0.5,
|
| 9 |
+
"fixed_t": null,
|
| 10 |
+
"max_len": 2048,
|
| 11 |
+
"batch_size_text": 8,
|
| 12 |
+
"io": "out",
|
| 13 |
+
"heldout_dataset": "common-pile/comma_v0.1_training_dataset",
|
| 14 |
+
"skip_first_n_examples": 500000
|
| 15 |
+
}
|