Upload folder using huggingface_hub

adapter_config.json

@@ -25,8 +25,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "v_proj"
+    "v_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c061a3917cf3a04e7dc66124efb6c9935579d2e24d2430c9be83832a4b443797
+oid sha256:c17095f6292d23f4591cde75ce6b9dee47c35287cff99daa7bf7d565c19918b7
 size 20992792

trainer_state.json

@@ -4,728 +4,878 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 471,
+  "global_step": 579,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.06369426751592357,
-      "grad_norm": 7.661201477050781,
-      "learning_rate": 9.80891719745223e-06,
-      "logits/chosen": -1.8285623788833618,
-      "logits/rejected": -1.613466501235962,
-      "logps/chosen": -513.4370727539062,
-      "logps/rejected": -476.92266845703125,
-      "loss": 0.7134,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.010444183833897114,
-      "rewards/margins": -0.03570808097720146,
-      "rewards/rejected": 0.02526390179991722,
+      "epoch": 0.05181347150259067,
+      "grad_norm": 9.140412330627441,
+      "learning_rate": 9.844559585492228e-06,
+      "logits/chosen": -1.9414482116699219,
+      "logits/rejected": -1.7225751876831055,
+      "logps/chosen": -486.18267822265625,
+      "logps/rejected": -488.115478515625,
+      "loss": 0.6963,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.023636549711227417,
+      "rewards/margins": -0.0014519490068778396,
+      "rewards/rejected": 0.02508850023150444,
       "step": 10
     },
     {
-      "epoch": 0.12738853503184713,
-      "grad_norm": 8.578171730041504,
-      "learning_rate": 9.596602972399152e-06,
-      "logits/chosen": -1.8371143341064453,
-      "logits/rejected": -1.5572088956832886,
-      "logps/chosen": -495.640869140625,
-      "logps/rejected": -517.6268310546875,
-      "loss": 0.6337,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.12352059036493301,
-      "rewards/margins": 0.12701454758644104,
-      "rewards/rejected": -0.003493957919999957,
+      "epoch": 0.10362694300518134,
+      "grad_norm": 8.910807609558105,
+      "learning_rate": 9.671848013816928e-06,
+      "logits/chosen": -1.8969703912734985,
+      "logits/rejected": -1.611445665359497,
+      "logps/chosen": -473.8232421875,
+      "logps/rejected": -541.8896484375,
+      "loss": 0.6356,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.09762275218963623,
+      "rewards/margins": 0.12276230752468109,
+      "rewards/rejected": -0.025139544159173965,
       "step": 20
     },
     {
-      "epoch": 0.1910828025477707,
-      "grad_norm": 8.120169639587402,
-      "learning_rate": 9.384288747346073e-06,
-      "logits/chosen": -1.794507622718811,
-      "logits/rejected": -1.6136707067489624,
-      "logps/chosen": -512.5538940429688,
-      "logps/rejected": -523.3352661132812,
-      "loss": 0.5442,
+      "epoch": 0.15544041450777202,
+      "grad_norm": 8.894061088562012,
+      "learning_rate": 9.499136442141624e-06,
+      "logits/chosen": -1.7466604709625244,
+      "logits/rejected": -1.5235388278961182,
+      "logps/chosen": -502.09619140625,
+      "logps/rejected": -496.20867919921875,
+      "loss": 0.5309,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2799207270145416,
-      "rewards/margins": 0.33490151166915894,
-      "rewards/rejected": -0.05498077720403671,
+      "rewards/chosen": 0.2354348599910736,
+      "rewards/margins": 0.36666831374168396,
+      "rewards/rejected": -0.13123343884944916,
       "step": 30
     },
     {
-      "epoch": 0.25477707006369427,
-      "grad_norm": 6.508784770965576,
-      "learning_rate": 9.171974522292994e-06,
-      "logits/chosen": -1.7962682247161865,
-      "logits/rejected": -1.6274349689483643,
-      "logps/chosen": -480.62139892578125,
-      "logps/rejected": -474.36273193359375,
-      "loss": 0.4204,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.4761812686920166,
-      "rewards/margins": 0.6683631539344788,
-      "rewards/rejected": -0.19218185544013977,
+      "epoch": 0.20725388601036268,
+      "grad_norm": 7.1635966300964355,
+      "learning_rate": 9.326424870466322e-06,
+      "logits/chosen": -1.8220049142837524,
+      "logits/rejected": -1.5709198713302612,
+      "logps/chosen": -486.42291259765625,
+      "logps/rejected": -516.5711669921875,
+      "loss": 0.4401,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.45994535088539124,
+      "rewards/margins": 0.6302233934402466,
+      "rewards/rejected": -0.17027801275253296,
       "step": 40
     },
     {
-      "epoch": 0.3184713375796178,
-      "grad_norm": 5.781623363494873,
-      "learning_rate": 8.959660297239916e-06,
-      "logits/chosen": -1.7765535116195679,
-      "logits/rejected": -1.591133713722229,
-      "logps/chosen": -507.0015563964844,
-      "logps/rejected": -491.17242431640625,
-      "loss": 0.327,
+      "epoch": 0.25906735751295334,
+      "grad_norm": 4.4454874992370605,
+      "learning_rate": 9.153713298791019e-06,
+      "logits/chosen": -1.7100648880004883,
+      "logits/rejected": -1.5425455570220947,
+      "logps/chosen": -521.1805419921875,
+      "logps/rejected": -486.8834533691406,
+      "loss": 0.2647,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.6968984007835388,
-      "rewards/margins": 0.9841921925544739,
-      "rewards/rejected": -0.28729376196861267,
+      "rewards/chosen": 0.9771218299865723,
+      "rewards/margins": 1.2796553373336792,
+      "rewards/rejected": -0.3025336265563965,
       "step": 50
     },
     {
-      "epoch": 0.3821656050955414,
-      "grad_norm": 2.6110823154449463,
-      "learning_rate": 8.747346072186837e-06,
-      "logits/chosen": -1.7369922399520874,
-      "logits/rejected": -1.5202242136001587,
-      "logps/chosen": -490.0065002441406,
-      "logps/rejected": -475.16748046875,
-      "loss": 0.2115,
+      "epoch": 0.31088082901554404,
+      "grad_norm": 4.048921585083008,
+      "learning_rate": 8.981001727115718e-06,
+      "logits/chosen": -1.6415131092071533,
+      "logits/rejected": -1.490295648574829,
+      "logps/chosen": -529.8136596679688,
+      "logps/rejected": -503.9295349121094,
+      "loss": 0.1952,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.994107723236084,
-      "rewards/margins": 1.5735362768173218,
-      "rewards/rejected": -0.579428493976593,
+      "rewards/chosen": 1.1363840103149414,
+      "rewards/margins": 1.6918432712554932,
+      "rewards/rejected": -0.5554592609405518,
       "step": 60
     },
     {
-      "epoch": 0.445859872611465,
-      "grad_norm": 3.15047287940979,
-      "learning_rate": 8.53503184713376e-06,
-      "logits/chosen": -1.6807092428207397,
-      "logits/rejected": -1.5059581995010376,
-      "logps/chosen": -487.39208984375,
-      "logps/rejected": -473.4630432128906,
-      "loss": 0.1456,
+      "epoch": 0.3626943005181347,
+      "grad_norm": 2.3416619300842285,
+      "learning_rate": 8.808290155440415e-06,
+      "logits/chosen": -1.6617555618286133,
+      "logits/rejected": -1.4604771137237549,
+      "logps/chosen": -514.0736694335938,
+      "logps/rejected": -523.543701171875,
+      "loss": 0.1252,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.3263133764266968,
-      "rewards/margins": 2.0210459232330322,
-      "rewards/rejected": -0.6947325468063354,
+      "rewards/chosen": 1.635059118270874,
+      "rewards/margins": 2.2532262802124023,
+      "rewards/rejected": -0.6181670427322388,
       "step": 70
     },
     {
-      "epoch": 0.5095541401273885,
-      "grad_norm": 2.4517059326171875,
-      "learning_rate": 8.32271762208068e-06,
-      "logits/chosen": -1.6148916482925415,
-      "logits/rejected": -1.3740451335906982,
-      "logps/chosen": -476.02166748046875,
-      "logps/rejected": -508.15887451171875,
-      "loss": 0.0904,
+      "epoch": 0.41450777202072536,
+      "grad_norm": 3.538511276245117,
+      "learning_rate": 8.635578583765113e-06,
+      "logits/chosen": -1.653364896774292,
+      "logits/rejected": -1.4298722743988037,
+      "logps/chosen": -474.07470703125,
+      "logps/rejected": -507.8360900878906,
+      "loss": 0.0758,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.7351124286651611,
-      "rewards/margins": 2.557111978530884,
-      "rewards/rejected": -0.8219996690750122,
+      "rewards/chosen": 1.8394428491592407,
+      "rewards/margins": 2.778176784515381,
+      "rewards/rejected": -0.9387339353561401,
       "step": 80
     },
     {
-      "epoch": 0.5732484076433121,
-      "grad_norm": 0.685248851776123,
-      "learning_rate": 8.110403397027601e-06,
-      "logits/chosen": -1.7143325805664062,
-      "logits/rejected": -1.481576919555664,
-      "logps/chosen": -483.890625,
-      "logps/rejected": -489.4288024902344,
-      "loss": 0.0508,
+      "epoch": 0.46632124352331605,
+      "grad_norm": 1.8926947116851807,
+      "learning_rate": 8.462867012089811e-06,
+      "logits/chosen": -1.6914196014404297,
+      "logits/rejected": -1.4301296472549438,
+      "logps/chosen": -478.93353271484375,
+      "logps/rejected": -479.542724609375,
+      "loss": 0.0446,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.919224500656128,
-      "rewards/margins": 3.2962067127227783,
-      "rewards/rejected": -1.3769820928573608,
+      "rewards/chosen": 2.3641176223754883,
+      "rewards/margins": 3.542463779449463,
+      "rewards/rejected": -1.1783459186553955,
       "step": 90
     },
     {
-      "epoch": 0.6369426751592356,
-      "grad_norm": 1.927983045578003,
-      "learning_rate": 7.898089171974524e-06,
-      "logits/chosen": -1.7103115320205688,
-      "logits/rejected": -1.4516280889511108,
-      "logps/chosen": -453.33721923828125,
-      "logps/rejected": -513.4710693359375,
-      "loss": 0.0314,
+      "epoch": 0.5181347150259067,
+      "grad_norm": 0.7793521881103516,
+      "learning_rate": 8.290155440414507e-06,
+      "logits/chosen": -1.6339620351791382,
+      "logits/rejected": -1.449350118637085,
+      "logps/chosen": -477.76446533203125,
+      "logps/rejected": -540.86328125,
+      "loss": 0.0393,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.1619415283203125,
-      "rewards/margins": 3.929858446121216,
-      "rewards/rejected": -1.7679170370101929,
+      "rewards/chosen": 2.5429415702819824,
+      "rewards/margins": 3.8789615631103516,
+      "rewards/rejected": -1.336020588874817,
       "step": 100
     },
     {
-      "epoch": 0.7006369426751592,
-      "grad_norm": 0.39471063017845154,
-      "learning_rate": 7.685774946921445e-06,
-      "logits/chosen": -1.6923818588256836,
-      "logits/rejected": -1.4913740158081055,
-      "logps/chosen": -487.2040100097656,
-      "logps/rejected": -534.4457397460938,
-      "loss": 0.0199,
+      "epoch": 0.5699481865284974,
+      "grad_norm": 0.5772690176963806,
+      "learning_rate": 8.117443868739207e-06,
+      "logits/chosen": -1.6833152770996094,
+      "logits/rejected": -1.3989613056182861,
+      "logps/chosen": -465.7315368652344,
+      "logps/rejected": -520.9325561523438,
+      "loss": 0.0211,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.3453280925750732,
-      "rewards/margins": 4.448094844818115,
-      "rewards/rejected": -2.102766513824463,
+      "rewards/chosen": 2.534696102142334,
+      "rewards/margins": 4.387997150421143,
+      "rewards/rejected": -1.85330069065094,
       "step": 110
     },
     {
-      "epoch": 0.7643312101910829,
-      "grad_norm": 0.1494450569152832,
-      "learning_rate": 7.473460721868365e-06,
-      "logits/chosen": -1.7227827310562134,
-      "logits/rejected": -1.5164122581481934,
-      "logps/chosen": -466.625244140625,
-      "logps/rejected": -496.8236389160156,
-      "loss": 0.0137,
+      "epoch": 0.6217616580310881,
+      "grad_norm": 0.4226108491420746,
+      "learning_rate": 7.944732297063904e-06,
+      "logits/chosen": -1.6814254522323608,
+      "logits/rejected": -1.4467347860336304,
+      "logps/chosen": -468.86248779296875,
+      "logps/rejected": -499.3148498535156,
+      "loss": 0.0157,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.604698419570923,
-      "rewards/margins": 4.873368740081787,
-      "rewards/rejected": -2.2686705589294434,
+      "rewards/chosen": 2.5799293518066406,
+      "rewards/margins": 4.886545181274414,
+      "rewards/rejected": -2.3066158294677734,
       "step": 120
     },
     {
-      "epoch": 0.8280254777070064,
-      "grad_norm": 0.979215681552887,
-      "learning_rate": 7.261146496815287e-06,
-      "logits/chosen": -1.6731090545654297,
-      "logits/rejected": -1.4788906574249268,
-      "logps/chosen": -489.17041015625,
-      "logps/rejected": -481.39910888671875,
-      "loss": 0.012,
+      "epoch": 0.6735751295336787,
+      "grad_norm": 0.3847591280937195,
+      "learning_rate": 7.772020725388602e-06,
+      "logits/chosen": -1.6606528759002686,
+      "logits/rejected": -1.418936014175415,
+      "logps/chosen": -479.26446533203125,
+      "logps/rejected": -526.3353271484375,
+      "loss": 0.0154,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.6250174045562744,
-      "rewards/margins": 5.168362617492676,
-      "rewards/rejected": -2.5433454513549805,
+      "rewards/chosen": 2.75547456741333,
+      "rewards/margins": 5.223846435546875,
+      "rewards/rejected": -2.468371868133545,
       "step": 130
     },
     {
-      "epoch": 0.89171974522293,
-      "grad_norm": 0.5591831207275391,
-      "learning_rate": 7.0488322717622086e-06,
-      "logits/chosen": -1.7174581289291382,
-      "logits/rejected": -1.491135835647583,
-      "logps/chosen": -494.3304748535156,
-      "logps/rejected": -539.1906127929688,
-      "loss": 0.0144,
+      "epoch": 0.7253886010362695,
+      "grad_norm": 0.2281242161989212,
+      "learning_rate": 7.599309153713299e-06,
+      "logits/chosen": -1.7155954837799072,
+      "logits/rejected": -1.4960612058639526,
+      "logps/chosen": -468.90350341796875,
+      "logps/rejected": -520.2388305664062,
+      "loss": 0.0087,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.630114793777466,
-      "rewards/margins": 5.534493923187256,
-      "rewards/rejected": -2.904378890991211,
+      "rewards/chosen": 2.7697935104370117,
+      "rewards/margins": 5.687216281890869,
+      "rewards/rejected": -2.917421817779541,
       "step": 140
     },
     {
-      "epoch": 0.9554140127388535,
-      "grad_norm": 0.10819283127784729,
-      "learning_rate": 6.836518046709129e-06,
-      "logits/chosen": -1.7261998653411865,
-      "logits/rejected": -1.5176118612289429,
-      "logps/chosen": -505.73779296875,
-      "logps/rejected": -508.22442626953125,
-      "loss": 0.0097,
+      "epoch": 0.7772020725388601,
+      "grad_norm": 0.508161187171936,
+      "learning_rate": 7.426597582037998e-06,
+      "logits/chosen": -1.689793348312378,
+      "logits/rejected": -1.4585435390472412,
+      "logps/chosen": -440.17803955078125,
+      "logps/rejected": -500.63897705078125,
+      "loss": 0.0107,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.8433146476745605,
-      "rewards/margins": 5.865386009216309,
-      "rewards/rejected": -3.022071123123169,
+      "rewards/chosen": 2.7548987865448,
+      "rewards/margins": 5.646756649017334,
+      "rewards/rejected": -2.891857624053955,
       "step": 150
     },
     {
-      "epoch": 1.019108280254777,
-      "grad_norm": 0.16133908927440643,
-      "learning_rate": 6.624203821656051e-06,
-      "logits/chosen": -1.8000129461288452,
-      "logits/rejected": -1.5112035274505615,
-      "logps/chosen": -472.0714416503906,
-      "logps/rejected": -514.2125854492188,
-      "loss": 0.0098,
+      "epoch": 0.8290155440414507,
+      "grad_norm": 0.1661311537027359,
+      "learning_rate": 7.253886010362695e-06,
+      "logits/chosen": -1.749815583229065,
+      "logits/rejected": -1.5069010257720947,
+      "logps/chosen": -484.00640869140625,
+      "logps/rejected": -507.42742919921875,
+      "loss": 0.0055,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.4917571544647217,
-      "rewards/margins": 5.523527145385742,
-      "rewards/rejected": -3.0317697525024414,
+      "rewards/chosen": 2.803175449371338,
+      "rewards/margins": 6.165472984313965,
+      "rewards/rejected": -3.362297534942627,
       "step": 160
     },
     {
-      "epoch": 1.0828025477707006,
-      "grad_norm": 0.14261530339717865,
-      "learning_rate": 6.411889596602973e-06,
-      "logits/chosen": -1.873085379600525,
-      "logits/rejected": -1.4990111589431763,
-      "logps/chosen": -462.21563720703125,
-      "logps/rejected": -520.8255004882812,
-      "loss": 0.0052,
+      "epoch": 0.8808290155440415,
+      "grad_norm": 0.19041810929775238,
+      "learning_rate": 7.0811744386873925e-06,
+      "logits/chosen": -1.694772481918335,
+      "logits/rejected": -1.460770606994629,
+      "logps/chosen": -479.5777282714844,
+      "logps/rejected": -535.0095825195312,
+      "loss": 0.007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.830996036529541,
-      "rewards/margins": 6.318862438201904,
-      "rewards/rejected": -3.4878666400909424,
+      "rewards/chosen": 2.6985023021698,
+      "rewards/margins": 6.264751434326172,
+      "rewards/rejected": -3.5662498474121094,
       "step": 170
     },
     {
-      "epoch": 1.1464968152866242,
-      "grad_norm": 0.11651387810707092,
-      "learning_rate": 6.199575371549894e-06,
-      "logits/chosen": -1.7707847356796265,
-      "logits/rejected": -1.548455834388733,
-      "logps/chosen": -516.4500732421875,
-      "logps/rejected": -549.6249389648438,
-      "loss": 0.0065,
+      "epoch": 0.9326424870466321,
+      "grad_norm": 0.2673930823802948,
+      "learning_rate": 6.90846286701209e-06,
+      "logits/chosen": -1.8051176071166992,
+      "logits/rejected": -1.5203473567962646,
+      "logps/chosen": -481.57830810546875,
+      "logps/rejected": -542.889404296875,
+      "loss": 0.0058,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.196195125579834,
-      "rewards/margins": 6.9291205406188965,
-      "rewards/rejected": -3.7329254150390625,
+      "rewards/chosen": 3.1937761306762695,
+      "rewards/margins": 6.7014665603637695,
+      "rewards/rejected": -3.507689952850342,
       "step": 180
     },
     {
-      "epoch": 1.2101910828025477,
-      "grad_norm": 2.9312522411346436,
-      "learning_rate": 5.987261146496816e-06,
-      "logits/chosen": -1.7615554332733154,
-      "logits/rejected": -1.5339546203613281,
-      "logps/chosen": -497.7105407714844,
-      "logps/rejected": -507.69140625,
-      "loss": 0.009,
+      "epoch": 0.9844559585492227,
+      "grad_norm": 0.5000354647636414,
+      "learning_rate": 6.735751295336788e-06,
+      "logits/chosen": -1.760645866394043,
+      "logits/rejected": -1.4444111585617065,
+      "logps/chosen": -480.0152282714844,
+      "logps/rejected": -560.7935791015625,
+      "loss": 0.0061,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.932077407836914,
-      "rewards/margins": 6.43866491317749,
-      "rewards/rejected": -3.506587505340576,
+      "rewards/chosen": 3.096393585205078,
+      "rewards/margins": 6.873165130615234,
+      "rewards/rejected": -3.7767722606658936,
       "step": 190
     },
     {
-      "epoch": 1.2738853503184713,
-      "grad_norm": 0.11866319924592972,
-      "learning_rate": 5.774946921443737e-06,
-      "logits/chosen": -1.8983533382415771,
-      "logits/rejected": -1.535003662109375,
-      "logps/chosen": -460.1544494628906,
-      "logps/rejected": -529.9498291015625,
-      "loss": 0.0018,
+      "epoch": 1.0362694300518134,
+      "grad_norm": 0.028729300945997238,
+      "learning_rate": 6.563039723661486e-06,
+      "logits/chosen": -1.768303632736206,
+      "logits/rejected": -1.4567553997039795,
+      "logps/chosen": -460.3399353027344,
+      "logps/rejected": -527.8641357421875,
+      "loss": 0.0041,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1939971446990967,
-      "rewards/margins": 7.317962646484375,
-      "rewards/rejected": -4.123966217041016,
+      "rewards/chosen": 2.6775355339050293,
+      "rewards/margins": 6.8611650466918945,
+      "rewards/rejected": -4.183629512786865,
       "step": 200
     },
     {
-      "epoch": 1.3375796178343948,
-      "grad_norm": 0.009582897648215294,
-      "learning_rate": 5.562632696390658e-06,
-      "logits/chosen": -1.8325570821762085,
-      "logits/rejected": -1.5961052179336548,
-      "logps/chosen": -457.8462829589844,
-      "logps/rejected": -511.46881103515625,
-      "loss": 0.0036,
+      "epoch": 1.0880829015544042,
+      "grad_norm": 0.03302822262048721,
+      "learning_rate": 6.390328151986184e-06,
+      "logits/chosen": -1.7313588857650757,
+      "logits/rejected": -1.4315208196640015,
+      "logps/chosen": -494.01141357421875,
+      "logps/rejected": -582.895751953125,
+      "loss": 0.0016,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.2415060997009277,
-      "rewards/margins": 7.01214599609375,
-      "rewards/rejected": -3.770639419555664,
+      "rewards/chosen": 3.1437973976135254,
+      "rewards/margins": 7.43361759185791,
+      "rewards/rejected": -4.289820194244385,
       "step": 210
     },
     {
-      "epoch": 1.4012738853503186,
-      "grad_norm": 0.2067570984363556,
-      "learning_rate": 5.35031847133758e-06,
-      "logits/chosen": -1.8559471368789673,
-      "logits/rejected": -1.4903216361999512,
-      "logps/chosen": -444.54339599609375,
-      "logps/rejected": -536.953369140625,
-      "loss": 0.0017,
+      "epoch": 1.1398963730569949,
+      "grad_norm": 0.2988791763782501,
+      "learning_rate": 6.217616580310881e-06,
+      "logits/chosen": -1.7174772024154663,
+      "logits/rejected": -1.4190794229507446,
+      "logps/chosen": -483.44439697265625,
+      "logps/rejected": -574.5506591796875,
+      "loss": 0.0049,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.150261163711548,
-      "rewards/margins": 7.365521430969238,
-      "rewards/rejected": -4.2152605056762695,
+      "rewards/chosen": 2.8517348766326904,
+      "rewards/margins": 7.021934509277344,
+      "rewards/rejected": -4.170199394226074,
       "step": 220
     },
     {
-      "epoch": 1.4649681528662422,
-      "grad_norm": 0.1383599489927292,
-      "learning_rate": 5.1380042462845016e-06,
-      "logits/chosen": -1.8445074558258057,
-      "logits/rejected": -1.6155668497085571,
-      "logps/chosen": -467.86846923828125,
-      "logps/rejected": -537.4169311523438,
-      "loss": 0.0023,
+      "epoch": 1.1917098445595855,
+      "grad_norm": 0.1659945398569107,
+      "learning_rate": 6.044905008635579e-06,
+      "logits/chosen": -1.8686014413833618,
+      "logits/rejected": -1.55970299243927,
+      "logps/chosen": -448.3949279785156,
+      "logps/rejected": -565.4742431640625,
+      "loss": 0.0038,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.124246835708618,
-      "rewards/margins": 7.138210296630859,
-      "rewards/rejected": -4.013963222503662,
+      "rewards/chosen": 3.227496385574341,
+      "rewards/margins": 7.409262180328369,
+      "rewards/rejected": -4.181765556335449,
       "step": 230
     },
     {
-      "epoch": 1.5286624203821657,
-      "grad_norm": 0.14673951268196106,
-      "learning_rate": 4.925690021231423e-06,
-      "logits/chosen": -1.7636497020721436,
-      "logits/rejected": -1.5512115955352783,
-      "logps/chosen": -480.15203857421875,
-      "logps/rejected": -529.8653564453125,
-      "loss": 0.0022,
+      "epoch": 1.2435233160621761,
+      "grad_norm": 0.0639355331659317,
+      "learning_rate": 5.872193436960278e-06,
+      "logits/chosen": -1.8060436248779297,
+      "logits/rejected": -1.545507550239563,
+      "logps/chosen": -478.74609375,
+      "logps/rejected": -555.3989868164062,
+      "loss": 0.0028,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.0038843154907227,
-      "rewards/margins": 7.056528568267822,
-      "rewards/rejected": -4.052645206451416,
+      "rewards/chosen": 3.2223987579345703,
+      "rewards/margins": 7.58017110824585,
+      "rewards/rejected": -4.357772350311279,
       "step": 240
     },
     {
-      "epoch": 1.5923566878980893,
-      "grad_norm": 0.19075194001197815,
-      "learning_rate": 4.713375796178344e-06,
-      "logits/chosen": -1.777608871459961,
-      "logits/rejected": -1.5940536260604858,
-      "logps/chosen": -492.18280029296875,
-      "logps/rejected": -511.8550720214844,
-      "loss": 0.0028,
+      "epoch": 1.2953367875647668,
+      "grad_norm": 1.1631646156311035,
+      "learning_rate": 5.699481865284975e-06,
+      "logits/chosen": -1.8868166208267212,
+      "logits/rejected": -1.5805160999298096,
+      "logps/chosen": -430.96728515625,
+      "logps/rejected": -526.8347778320312,
+      "loss": 0.0034,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.5837788581848145,
-      "rewards/margins": 6.94525146484375,
-      "rewards/rejected": -4.361472129821777,
+      "rewards/chosen": 3.246020555496216,
+      "rewards/margins": 8.115084648132324,
+      "rewards/rejected": -4.869065284729004,
       "step": 250
     },
     {
-      "epoch": 1.6560509554140128,
-      "grad_norm": 0.10456167906522751,
-      "learning_rate": 4.501061571125266e-06,
-      "logits/chosen": -1.8056246042251587,
-      "logits/rejected": -1.5386626720428467,
-      "logps/chosen": -488.4483337402344,
-      "logps/rejected": -552.2747192382812,
-      "loss": 0.0017,
+      "epoch": 1.3471502590673574,
+      "grad_norm": 0.03497570380568504,
+      "learning_rate": 5.526770293609672e-06,
+      "logits/chosen": -1.7296884059906006,
+      "logits/rejected": -1.4742090702056885,
+      "logps/chosen": -495.16925048828125,
+      "logps/rejected": -553.0023803710938,
+      "loss": 0.0025,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.2254958152770996,
-      "rewards/margins": 7.798495292663574,
-      "rewards/rejected": -4.572999477386475,
+      "rewards/chosen": 3.5322353839874268,
+      "rewards/margins": 7.728882789611816,
+      "rewards/rejected": -4.196647644042969,
       "step": 260
     },
     {
-      "epoch": 1.7197452229299364,
-      "grad_norm": 0.08734221756458282,
-      "learning_rate": 4.288747346072187e-06,
-      "logits/chosen": -1.8739216327667236,
-      "logits/rejected": -1.5936236381530762,
-      "logps/chosen": -481.37353515625,
-      "logps/rejected": -506.7474670410156,
-      "loss": 0.0041,
+      "epoch": 1.3989637305699483,
+      "grad_norm": 0.07264388352632523,
+      "learning_rate": 5.3540587219343694e-06,
+      "logits/chosen": -1.8094289302825928,
+      "logits/rejected": -1.5321930646896362,
+      "logps/chosen": -463.7430725097656,
+      "logps/rejected": -537.8118896484375,
+      "loss": 0.0008,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.923518180847168,
-      "rewards/margins": 7.2085113525390625,
-      "rewards/rejected": -4.284992218017578,
+      "rewards/chosen": 3.354356288909912,
+      "rewards/margins": 8.136880874633789,
+      "rewards/rejected": -4.782524585723877,
       "step": 270
     },
     {
-      "epoch": 1.78343949044586,
-      "grad_norm": 0.06190163269639015,
-      "learning_rate": 4.076433121019109e-06,
-      "logits/chosen": -1.8493846654891968,
-      "logits/rejected": -1.535466194152832,
-      "logps/chosen": -465.69134521484375,
-      "logps/rejected": -524.5106201171875,
-      "loss": 0.0013,
+      "epoch": 1.450777202072539,
+      "grad_norm": 0.008589176461100578,
+      "learning_rate": 5.1813471502590676e-06,
+      "logits/chosen": -1.7734190225601196,
+      "logits/rejected": -1.5649479627609253,
+      "logps/chosen": -458.47503662109375,
+      "logps/rejected": -526.0906372070312,
+      "loss": 0.0023,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.9931843280792236,
-      "rewards/margins": 7.830008506774902,
-      "rewards/rejected": -4.836824893951416,
+      "rewards/chosen": 3.3651351928710938,
+      "rewards/margins": 8.263564109802246,
+      "rewards/rejected": -4.898428916931152,
       "step": 280
     },
     {
-      "epoch": 1.8471337579617835,
-      "grad_norm": 0.056305430829524994,
-      "learning_rate": 3.8641188959660305e-06,
-      "logits/chosen": -1.9110437631607056,
-      "logits/rejected": -1.5337971448898315,
-      "logps/chosen": -461.9651794433594,
-      "logps/rejected": -535.5386962890625,
-      "loss": 0.0009,
+      "epoch": 1.5025906735751295,
+      "grad_norm": 0.04104425758123398,
+      "learning_rate": 5.008635578583766e-06,
+      "logits/chosen": -1.878761649131775,
+      "logits/rejected": -1.4834644794464111,
+      "logps/chosen": -461.1991271972656,
+      "logps/rejected": -540.1414794921875,
+      "loss": 0.0012,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1686594486236572,
-      "rewards/margins": 7.800524711608887,
-      "rewards/rejected": -4.63186502456665,
+      "rewards/chosen": 3.5035297870635986,
+      "rewards/margins": 8.194310188293457,
+      "rewards/rejected": -4.690779685974121,
       "step": 290
     },
     {
-      "epoch": 1.910828025477707,
-      "grad_norm": 0.07272353023290634,
-      "learning_rate": 3.6518046709129513e-06,
-      "logits/chosen": -1.8243738412857056,
-      "logits/rejected": -1.5750045776367188,
-      "logps/chosen": -457.2413635253906,
-      "logps/rejected": -554.6361083984375,
-      "loss": 0.0011,
+      "epoch": 1.5544041450777202,
+      "grad_norm": 0.01376403495669365,
+      "learning_rate": 4.835924006908464e-06,
+      "logits/chosen": -1.888736367225647,
+      "logits/rejected": -1.5105645656585693,
+      "logps/chosen": -470.4767150878906,
+      "logps/rejected": -569.6588745117188,
+      "loss": 0.0021,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1141717433929443,
-      "rewards/margins": 7.966392517089844,
-      "rewards/rejected": -4.852221488952637,
+      "rewards/chosen": 3.319579601287842,
+      "rewards/margins": 8.164112091064453,
+      "rewards/rejected": -4.8445329666137695,
       "step": 300
     },
     {
-      "epoch": 1.9745222929936306,
-      "grad_norm": 0.1764669567346573,
-      "learning_rate": 3.4394904458598725e-06,
-      "logits/chosen": -1.8024390935897827,
-      "logits/rejected": -1.529760479927063,
-      "logps/chosen": -482.1455078125,
-      "logps/rejected": -523.8703002929688,
-      "loss": 0.0026,
+      "epoch": 1.6062176165803108,
+      "grad_norm": 0.3114416003227234,
+      "learning_rate": 4.663212435233161e-06,
+      "logits/chosen": -1.85044264793396,
+      "logits/rejected": -1.5581977367401123,
+      "logps/chosen": -475.26690673828125,
+      "logps/rejected": -520.6973876953125,
+      "loss": 0.0018,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.9039502143859863,
-      "rewards/margins": 7.471547603607178,
-      "rewards/rejected": -4.56759786605835,
+      "rewards/chosen": 3.5499215126037598,
+      "rewards/margins": 8.229494094848633,
+      "rewards/rejected": -4.679572105407715,
       "step": 310
     },
     {
-      "epoch": 2.038216560509554,
-      "grad_norm": 0.06449217349290848,
-      "learning_rate": 3.227176220806794e-06,
-      "logits/chosen": -1.8698956966400146,
-      "logits/rejected": -1.5352425575256348,
-      "logps/chosen": -474.93487548828125,
-      "logps/rejected": -555.2481689453125,
-      "loss": 0.003,
+      "epoch": 1.6580310880829017,
+      "grad_norm": 0.08180935680866241,
+      "learning_rate": 4.490500863557859e-06,
+      "logits/chosen": -1.8729044198989868,
+      "logits/rejected": -1.6026229858398438,
+      "logps/chosen": -452.8746032714844,
+      "logps/rejected": -530.5598754882812,
+      "loss": 0.0007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.7326927185058594,
-      "rewards/margins": 7.607143402099609,
-      "rewards/rejected": -4.874450206756592,
+      "rewards/chosen": 3.1103663444519043,
+      "rewards/margins": 8.12248420715332,
+      "rewards/rejected": -5.012118339538574,
       "step": 320
     },
     {
-      "epoch": 2.1019108280254777,
-      "grad_norm": 0.15638966858386993,
-      "learning_rate": 3.0148619957537158e-06,
-      "logits/chosen": -1.8096567392349243,
-      "logits/rejected": -1.5850298404693604,
-      "logps/chosen": -498.39483642578125,
-      "logps/rejected": -536.6939086914062,
+      "epoch": 1.709844559585492,
+      "grad_norm": 0.03724868223071098,
+      "learning_rate": 4.3177892918825564e-06,
+      "logits/chosen": -1.8127027750015259,
+      "logits/rejected": -1.5433104038238525,
+      "logps/chosen": -510.96234130859375,
+      "logps/rejected": -539.6664428710938,
       "loss": 0.0011,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.083881139755249,
-      "rewards/margins": 8.246601104736328,
-      "rewards/rejected": -5.162720203399658,
+      "rewards/chosen": 3.440533399581909,
+      "rewards/margins": 8.182977676391602,
+      "rewards/rejected": -4.742444038391113,
       "step": 330
     },
     {
-      "epoch": 2.1656050955414012,
-      "grad_norm": 0.022478261962532997,
-      "learning_rate": 2.802547770700637e-06,
-      "logits/chosen": -1.8860228061676025,
-      "logits/rejected": -1.5795822143554688,
-      "logps/chosen": -471.2503356933594,
-      "logps/rejected": -555.7251586914062,
-      "loss": 0.001,
+      "epoch": 1.761658031088083,
+      "grad_norm": 0.024448685348033905,
+      "learning_rate": 4.145077720207254e-06,
+      "logits/chosen": -1.9371646642684937,
+      "logits/rejected": -1.5605006217956543,
+      "logps/chosen": -457.09521484375,
+      "logps/rejected": -562.9553833007812,
+      "loss": 0.0005,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.8919870853424072,
-      "rewards/margins": 8.094644546508789,
-      "rewards/rejected": -5.2026591300964355,
+      "rewards/chosen": 3.8051505088806152,
+      "rewards/margins": 8.683636665344238,
+      "rewards/rejected": -4.878485679626465,
       "step": 340
     },
     {
-      "epoch": 2.229299363057325,
-      "grad_norm": 0.006824295036494732,
-      "learning_rate": 2.5902335456475586e-06,
-      "logits/chosen": -1.840765357017517,
-      "logits/rejected": -1.6004451513290405,
-      "logps/chosen": -496.37908935546875,
-      "logps/rejected": -516.0117797851562,
-      "loss": 0.0012,
+      "epoch": 1.8134715025906736,
+      "grad_norm": 0.007909624837338924,
+      "learning_rate": 3.972366148531952e-06,
+      "logits/chosen": -1.7813365459442139,
+      "logits/rejected": -1.5386850833892822,
+      "logps/chosen": -472.1241760253906,
+      "logps/rejected": -526.0204467773438,
+      "loss": 0.0008,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.2698845863342285,
-      "rewards/margins": 8.031291007995605,
-      "rewards/rejected": -4.761406898498535,
+      "rewards/chosen": 3.475656032562256,
+      "rewards/margins": 8.600740432739258,
+      "rewards/rejected": -5.125083923339844,
       "step": 350
     },
     {
-      "epoch": 2.2929936305732483,
-      "grad_norm": 0.041482195258140564,
-      "learning_rate": 2.3779193205944802e-06,
-      "logits/chosen": -1.9090330600738525,
-      "logits/rejected": -1.5509936809539795,
-      "logps/chosen": -476.31884765625,
-      "logps/rejected": -530.485595703125,
-      "loss": 0.0014,
+      "epoch": 1.8652849740932642,
+      "grad_norm": 0.01100204512476921,
+      "learning_rate": 3.7996545768566495e-06,
+      "logits/chosen": -1.8551228046417236,
+      "logits/rejected": -1.6169090270996094,
+      "logps/chosen": -468.3399353027344,
+      "logps/rejected": -521.0477294921875,
+      "loss": 0.0022,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.3065025806427,
-      "rewards/margins": 8.122949600219727,
-      "rewards/rejected": -4.816446304321289,
+      "rewards/chosen": 3.692009449005127,
+      "rewards/margins": 8.327119827270508,
+      "rewards/rejected": -4.635110855102539,
       "step": 360
     },
     {
-      "epoch": 2.356687898089172,
-      "grad_norm": 0.02308792807161808,
-      "learning_rate": 2.1656050955414015e-06,
-      "logits/chosen": -1.887399435043335,
-      "logits/rejected": -1.5576483011245728,
-      "logps/chosen": -468.9479064941406,
-      "logps/rejected": -581.8629150390625,
-      "loss": 0.0014,
+      "epoch": 1.917098445595855,
+      "grad_norm": 0.002755044959485531,
+      "learning_rate": 3.6269430051813476e-06,
+      "logits/chosen": -1.920689582824707,
+      "logits/rejected": -1.5247467756271362,
+      "logps/chosen": -458.05718994140625,
+      "logps/rejected": -549.6717529296875,
+      "loss": 0.0007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.292541980743408,
-      "rewards/margins": 8.23143196105957,
-      "rewards/rejected": -4.938891410827637,
+      "rewards/chosen": 3.8437609672546387,
+      "rewards/margins": 8.824178695678711,
+      "rewards/rejected": -4.980417251586914,
       "step": 370
     },
     {
-      "epoch": 2.4203821656050954,
-      "grad_norm": 0.06411473453044891,
-      "learning_rate": 1.953290870488323e-06,
-      "logits/chosen": -1.82333242893219,
-      "logits/rejected": -1.67373788356781,
-      "logps/chosen": -483.3836975097656,
-      "logps/rejected": -511.4134216308594,
-      "loss": 0.001,
+      "epoch": 1.9689119170984455,
+      "grad_norm": 0.001801456674002111,
+      "learning_rate": 3.454231433506045e-06,
+      "logits/chosen": -1.819500207901001,
+      "logits/rejected": -1.5786559581756592,
+      "logps/chosen": -478.22540283203125,
+      "logps/rejected": -571.024658203125,
+      "loss": 0.0006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.0308749675750732,
-      "rewards/margins": 8.01519775390625,
-      "rewards/rejected": -4.984322547912598,
+      "rewards/chosen": 3.27850079536438,
+      "rewards/margins": 8.988458633422852,
+      "rewards/rejected": -5.709958076477051,
       "step": 380
     },
     {
-      "epoch": 2.484076433121019,
-      "grad_norm": 0.20943918824195862,
-      "learning_rate": 1.740976645435244e-06,
-      "logits/chosen": -1.8794472217559814,
-      "logits/rejected": -1.5557177066802979,
-      "logps/chosen": -476.9034118652344,
-      "logps/rejected": -540.9918212890625,
-      "loss": 0.0015,
+      "epoch": 2.0207253886010363,
+      "grad_norm": 0.1213519424200058,
+      "learning_rate": 3.281519861830743e-06,
+      "logits/chosen": -1.7965360879898071,
+      "logits/rejected": -1.503381609916687,
+      "logps/chosen": -484.7544860839844,
+      "logps/rejected": -558.208740234375,
+      "loss": 0.0019,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.99308443069458,
-      "rewards/margins": 7.622200965881348,
-      "rewards/rejected": -4.629117012023926,
+      "rewards/chosen": 3.11090087890625,
+      "rewards/margins": 8.451322555541992,
+      "rewards/rejected": -5.340420722961426,
       "step": 390
     },
     {
-      "epoch": 2.5477707006369426,
-      "grad_norm": 0.15412284433841705,
-      "learning_rate": 1.5286624203821657e-06,
-      "logits/chosen": -1.8760645389556885,
-      "logits/rejected": -1.596841812133789,
-      "logps/chosen": -473.80413818359375,
-      "logps/rejected": -522.9478759765625,
-      "loss": 0.0007,
+      "epoch": 2.0725388601036268,
+      "grad_norm": 0.033907659351825714,
+      "learning_rate": 3.1088082901554407e-06,
+      "logits/chosen": -1.9653284549713135,
+      "logits/rejected": -1.5873863697052002,
+      "logps/chosen": -445.7735290527344,
+      "logps/rejected": -544.8649291992188,
+      "loss": 0.0003,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.289515972137451,
-      "rewards/margins": 8.1244478225708,
-      "rewards/rejected": -4.834931373596191,
+      "rewards/chosen": 3.4412612915039062,
+      "rewards/margins": 9.170848846435547,
+      "rewards/rejected": -5.729588031768799,
       "step": 400
     },
     {
-      "epoch": 2.611464968152866,
-      "grad_norm": 0.020231205970048904,
-      "learning_rate": 1.3163481953290871e-06,
-      "logits/chosen": -1.9735329151153564,
-      "logits/rejected": -1.6164305210113525,
-      "logps/chosen": -445.96075439453125,
-      "logps/rejected": -513.9588623046875,
-      "loss": 0.0014,
+      "epoch": 2.1243523316062176,
+      "grad_norm": 0.006869920063763857,
+      "learning_rate": 2.936096718480139e-06,
+      "logits/chosen": -1.8688383102416992,
+      "logits/rejected": -1.613979697227478,
+      "logps/chosen": -463.5596618652344,
+      "logps/rejected": -509.37579345703125,
+      "loss": 0.0013,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1464686393737793,
-      "rewards/margins": 8.033473014831543,
-      "rewards/rejected": -4.887004375457764,
+      "rewards/chosen": 3.1676273345947266,
+      "rewards/margins": 8.486566543579102,
+      "rewards/rejected": -5.318938732147217,
       "step": 410
     },
     {
-      "epoch": 2.6751592356687897,
-      "grad_norm": 0.06227200850844383,
-      "learning_rate": 1.1040339702760086e-06,
-      "logits/chosen": -1.883143424987793,
-      "logits/rejected": -1.5652716159820557,
-      "logps/chosen": -454.7635803222656,
-      "logps/rejected": -521.7005004882812,
-      "loss": 0.0018,
+      "epoch": 2.1761658031088085,
+      "grad_norm": 0.016242429614067078,
+      "learning_rate": 2.763385146804836e-06,
+      "logits/chosen": -1.8480716943740845,
+      "logits/rejected": -1.5438343286514282,
+      "logps/chosen": -480.6341857910156,
+      "logps/rejected": -562.0118408203125,
+      "loss": 0.0004,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.9739201068878174,
-      "rewards/margins": 7.8255157470703125,
-      "rewards/rejected": -4.851595878601074,
+      "rewards/chosen": 3.927326202392578,
+      "rewards/margins": 9.179906845092773,
+      "rewards/rejected": -5.252579689025879,
       "step": 420
     },
     {
-      "epoch": 2.738853503184713,
-      "grad_norm": 0.20386821031570435,
-      "learning_rate": 8.9171974522293e-07,
-      "logits/chosen": -1.8953783512115479,
-      "logits/rejected": -1.6141021251678467,
-      "logps/chosen": -455.4198303222656,
-      "logps/rejected": -504.2140197753906,
-      "loss": 0.0017,
+      "epoch": 2.227979274611399,
+      "grad_norm": 0.01917063444852829,
+      "learning_rate": 2.5906735751295338e-06,
+      "logits/chosen": -1.8709628582000732,
+      "logits/rejected": -1.5309154987335205,
+      "logps/chosen": -454.34661865234375,
+      "logps/rejected": -585.8658447265625,
+      "loss": 0.0011,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1860811710357666,
-      "rewards/margins": 7.73357629776001,
-      "rewards/rejected": -4.547495365142822,
+      "rewards/chosen": 3.5671658515930176,
+      "rewards/margins": 8.807685852050781,
+      "rewards/rejected": -5.2405195236206055,
       "step": 430
     },
     {
-      "epoch": 2.802547770700637,
-      "grad_norm": 0.036846332252025604,
-      "learning_rate": 6.794055201698514e-07,
-      "logits/chosen": -1.8413372039794922,
-      "logits/rejected": -1.507743000984192,
-      "logps/chosen": -455.23681640625,
-      "logps/rejected": -575.4034423828125,
-      "loss": 0.0006,
+      "epoch": 2.2797927461139897,
+      "grad_norm": 0.011940844357013702,
+      "learning_rate": 2.417962003454232e-06,
+      "logits/chosen": -1.8891462087631226,
+      "logits/rejected": -1.5786548852920532,
+      "logps/chosen": -476.75482177734375,
+      "logps/rejected": -550.165771484375,
+      "loss": 0.001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.36517596244812,
-      "rewards/margins": 8.604715347290039,
-      "rewards/rejected": -5.239538192749023,
+      "rewards/chosen": 3.5058720111846924,
+      "rewards/margins": 8.823278427124023,
+      "rewards/rejected": -5.317407131195068,
       "step": 440
     },
     {
-      "epoch": 2.8662420382165603,
-      "grad_norm": 0.0049489871598780155,
-      "learning_rate": 4.6709129511677283e-07,
-      "logits/chosen": -1.9412555694580078,
-      "logits/rejected": -1.5940210819244385,
-      "logps/chosen": -454.66143798828125,
-      "logps/rejected": -539.107177734375,
-      "loss": 0.0019,
+      "epoch": 2.33160621761658,
+      "grad_norm": 0.052170418202877045,
+      "learning_rate": 2.2452504317789296e-06,
+      "logits/chosen": -1.851637840270996,
+      "logits/rejected": -1.586336374282837,
+      "logps/chosen": -470.76483154296875,
+      "logps/rejected": -554.5698852539062,
+      "loss": 0.0008,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.9391071796417236,
-      "rewards/margins": 8.104011535644531,
-      "rewards/rejected": -5.1649041175842285,
+      "rewards/chosen": 2.9407029151916504,
+      "rewards/margins": 8.625204086303711,
+      "rewards/rejected": -5.6845011711120605,
       "step": 450
     },
     {
-      "epoch": 2.9299363057324843,
-      "grad_norm": 0.014217167161405087,
-      "learning_rate": 2.547770700636943e-07,
-      "logits/chosen": -1.8514007329940796,
-      "logits/rejected": -1.5744428634643555,
-      "logps/chosen": -488.06982421875,
-      "logps/rejected": -548.7665405273438,
-      "loss": 0.0012,
+      "epoch": 2.383419689119171,
+      "grad_norm": 0.003704603761434555,
+      "learning_rate": 2.072538860103627e-06,
+      "logits/chosen": -1.9217126369476318,
+      "logits/rejected": -1.5532411336898804,
+      "logps/chosen": -446.28118896484375,
+      "logps/rejected": -548.06982421875,
+      "loss": 0.0006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1351876258850098,
-      "rewards/margins": 8.055967330932617,
-      "rewards/rejected": -4.920780181884766,
+      "rewards/chosen": 3.3570828437805176,
+      "rewards/margins": 8.866881370544434,
+      "rewards/rejected": -5.509798049926758,
       "step": 460
     },
     {
-      "epoch": 2.9936305732484074,
-      "grad_norm": 0.030622508376836777,
-      "learning_rate": 4.246284501061571e-08,
-      "logits/chosen": -1.9187486171722412,
-      "logits/rejected": -1.607115387916565,
-      "logps/chosen": -481.8365173339844,
-      "logps/rejected": -552.6552734375,
-      "loss": 0.001,
+      "epoch": 2.4352331606217614,
+      "grad_norm": 0.12578149139881134,
+      "learning_rate": 1.8998272884283248e-06,
+      "logits/chosen": -1.7785208225250244,
+      "logits/rejected": -1.5287562608718872,
+      "logps/chosen": -507.2691955566406,
+      "logps/rejected": -557.8547973632812,
+      "loss": 0.0017,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.1875436305999756,
-      "rewards/margins": 8.592950820922852,
-      "rewards/rejected": -5.405407905578613,
+      "rewards/chosen": 3.180009365081787,
+      "rewards/margins": 8.626852989196777,
+      "rewards/rejected": -5.446843147277832,
       "step": 470
     },
+    {
+      "epoch": 2.4870466321243523,
+      "grad_norm": 0.04742634296417236,
+      "learning_rate": 1.7271157167530224e-06,
+      "logits/chosen": -1.7439069747924805,
+      "logits/rejected": -1.504612922668457,
+      "logps/chosen": -492.09442138671875,
+      "logps/rejected": -556.8635864257812,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.334986448287964,
+      "rewards/margins": 8.468341827392578,
+      "rewards/rejected": -5.133355140686035,
+      "step": 480
+    },
+    {
+      "epoch": 2.538860103626943,
+      "grad_norm": 0.005521442741155624,
+      "learning_rate": 1.5544041450777204e-06,
+      "logits/chosen": -1.9542961120605469,
+      "logits/rejected": -1.5555959939956665,
+      "logps/chosen": -452.74365234375,
+      "logps/rejected": -573.016357421875,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 4.025125026702881,
+      "rewards/margins": 9.435617446899414,
+      "rewards/rejected": -5.410491466522217,
+      "step": 490
+    },
+    {
+      "epoch": 2.5906735751295336,
+      "grad_norm": 0.004860945511609316,
+      "learning_rate": 1.381692573402418e-06,
+      "logits/chosen": -1.9969520568847656,
+      "logits/rejected": -1.6214845180511475,
+      "logps/chosen": -430.89959716796875,
+      "logps/rejected": -567.4713134765625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.1888136863708496,
+      "rewards/margins": 8.952463150024414,
+      "rewards/rejected": -5.763648509979248,
+      "step": 500
+    },
+    {
+      "epoch": 2.6424870466321244,
+      "grad_norm": 0.015202810987830162,
+      "learning_rate": 1.208981001727116e-06,
+      "logits/chosen": -1.8306211233139038,
+      "logits/rejected": -1.5266889333724976,
+      "logps/chosen": -476.01239013671875,
+      "logps/rejected": -548.576171875,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.322007417678833,
+      "rewards/margins": 8.947591781616211,
+      "rewards/rejected": -5.625583648681641,
+      "step": 510
+    },
+    {
+      "epoch": 2.694300518134715,
+      "grad_norm": 0.0031490169931203127,
+      "learning_rate": 1.0362694300518134e-06,
+      "logits/chosen": -1.7937307357788086,
+      "logits/rejected": -1.5639219284057617,
+      "logps/chosen": -476.2064514160156,
+      "logps/rejected": -544.3578491210938,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.444533586502075,
+      "rewards/margins": 8.885669708251953,
+      "rewards/rejected": -5.441136837005615,
+      "step": 520
+    },
+    {
+      "epoch": 2.7461139896373057,
+      "grad_norm": 0.009058469906449318,
+      "learning_rate": 8.635578583765112e-07,
+      "logits/chosen": -1.9383313655853271,
+      "logits/rejected": -1.6197090148925781,
+      "logps/chosen": -452.0750427246094,
+      "logps/rejected": -558.096435546875,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.907386064529419,
+      "rewards/margins": 9.26667594909668,
+      "rewards/rejected": -5.359290599822998,
+      "step": 530
+    },
+    {
+      "epoch": 2.7979274611398965,
+      "grad_norm": 0.001649867626838386,
+      "learning_rate": 6.90846286701209e-07,
+      "logits/chosen": -1.7749992609024048,
+      "logits/rejected": -1.5281288623809814,
+      "logps/chosen": -506.68621826171875,
+      "logps/rejected": -570.96630859375,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.9603734016418457,
+      "rewards/margins": 9.308938026428223,
+      "rewards/rejected": -5.348565101623535,
+      "step": 540
+    },
+    {
+      "epoch": 2.849740932642487,
+      "grad_norm": 0.1349392831325531,
+      "learning_rate": 5.181347150259067e-07,
+      "logits/chosen": -1.9128179550170898,
+      "logits/rejected": -1.5522171258926392,
+      "logps/chosen": -459.3995666503906,
+      "logps/rejected": -547.3722534179688,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.1473488807678223,
+      "rewards/margins": 8.742547988891602,
+      "rewards/rejected": -5.595198631286621,
+      "step": 550
+    },
+    {
+      "epoch": 2.901554404145078,
+      "grad_norm": 0.0068402839824557304,
+      "learning_rate": 3.454231433506045e-07,
+      "logits/chosen": -1.833030104637146,
+      "logits/rejected": -1.5440208911895752,
+      "logps/chosen": -476.62060546875,
+      "logps/rejected": -545.6508178710938,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.1847169399261475,
+      "rewards/margins": 9.365403175354004,
+      "rewards/rejected": -6.180685997009277,
+      "step": 560
+    },
+    {
+      "epoch": 2.9533678756476682,
+      "grad_norm": 0.009579150937497616,
+      "learning_rate": 1.7271157167530226e-07,
+      "logits/chosen": -1.7974345684051514,
+      "logits/rejected": -1.507267713546753,
+      "logps/chosen": -483.2386779785156,
+      "logps/rejected": -540.8428955078125,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.946927070617676,
+      "rewards/margins": 8.679272651672363,
+      "rewards/rejected": -5.732344627380371,
+      "step": 570
+    },
     {
       "epoch": 3.0,
-      "step": 471,
+      "step": 579,
       "total_flos": 0.0,
-      "train_loss": 0.07041998074037535,
-      "train_runtime": 1050.6186,
-      "train_samples_per_second": 1.79,
-      "train_steps_per_second": 0.448
+      "train_loss": 0.055241224655177,
+      "train_runtime": 1305.5765,
+      "train_samples_per_second": 1.774,
+      "train_steps_per_second": 0.443
     }
   ],
   "logging_steps": 10,
-  "max_steps": 471,
+  "max_steps": 579,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:103731271f099b914bbf8a25222d8c2b595695ca7764d33196520cc1ba53a4df
+oid sha256:a9e903857df414eaf4d2e8deceb20c7a0c7f1ac3a16d6cd4a85bd6c37fa86ea8
 size 6929