Upload folder using huggingface_hub

README.md

@@ -38,10 +38,10 @@ This model was trained with DPO, a method introduced in [Direct Preference Optim
 ### Framework versions
 
 - PEFT 0.17.1
-- TRL: 0.23.1
-- Transformers: 4.57.0
-- Pytorch: 2.8.0
-- Datasets: 4.1.1
+- TRL: 0.24.0
+- Transformers: 4.57.1
+- Pytorch: 2.9.0
+- Datasets: 4.2.0
 - Tokenizers: 0.22.1
 
 ## Citations

adapter_config.json

@@ -25,8 +25,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj"
+    "q_proj",
+    "v_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6db4bd01e3ba164b19acfbcde1b1672b273d4c4922cb841ef72cb84d54a3c9f
+oid sha256:22fe8e2ca50c8fdfaeadbee04d86b2bd4084c70d1135882db04e89e9fb2db999
 size 20992792

trainer_state.json

@@ -4,128 +4,23 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 72,
+  "global_step": 6,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.4166666666666667,
-      "grad_norm": 5.378512382507324,
-      "learning_rate": 8.750000000000001e-06,
-      "logits/chosen": -1.844266653060913,
-      "logits/rejected": -1.6856653690338135,
-      "logps/chosen": -486.0516662597656,
-      "logps/rejected": -451.41192626953125,
-      "loss": 0.6871,
-      "rewards/accuracies": 0.49000000953674316,
-      "rewards/chosen": 0.025482425466179848,
-      "rewards/margins": 0.01759222522377968,
-      "rewards/rejected": 0.007890196517109871,
-      "step": 10
-    },
-    {
-      "epoch": 0.8333333333333334,
-      "grad_norm": 6.4273881912231445,
-      "learning_rate": 7.361111111111112e-06,
-      "logits/chosen": -1.8168048858642578,
-      "logits/rejected": -1.6364498138427734,
-      "logps/chosen": -486.64166259765625,
-      "logps/rejected": -463.81634521484375,
-      "loss": 0.6185,
-      "rewards/accuracies": 0.8799999952316284,
-      "rewards/chosen": 0.10465441644191742,
-      "rewards/margins": 0.16221235692501068,
-      "rewards/rejected": -0.05755792185664177,
-      "step": 20
-    },
-    {
-      "epoch": 1.25,
-      "grad_norm": 5.65267276763916,
-      "learning_rate": 5.972222222222222e-06,
-      "logits/chosen": -1.8458976745605469,
-      "logits/rejected": -1.6958911418914795,
-      "logps/chosen": -486.38079833984375,
-      "logps/rejected": -460.72991943359375,
-      "loss": 0.5097,
-      "rewards/accuracies": 0.9899999499320984,
-      "rewards/chosen": 0.2503821849822998,
-      "rewards/margins": 0.4185231626033783,
-      "rewards/rejected": -0.1681409627199173,
-      "step": 30
-    },
-    {
-      "epoch": 1.6666666666666665,
-      "grad_norm": 4.828741550445557,
-      "learning_rate": 4.583333333333333e-06,
-      "logits/chosen": -1.7737739086151123,
-      "logits/rejected": -1.615502953529358,
-      "logps/chosen": -503.5326232910156,
-      "logps/rejected": -456.4029235839844,
-      "loss": 0.418,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.41943544149398804,
-      "rewards/margins": 0.6772049069404602,
-      "rewards/rejected": -0.2577693462371826,
-      "step": 40
-    },
-    {
-      "epoch": 2.0833333333333335,
-      "grad_norm": 4.443270683288574,
-      "learning_rate": 3.1944444444444443e-06,
-      "logits/chosen": -1.8288230895996094,
-      "logits/rejected": -1.6355512142181396,
-      "logps/chosen": -478.97039794921875,
-      "logps/rejected": -471.06170654296875,
-      "loss": 0.3435,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.5154451727867126,
-      "rewards/margins": 0.9199325442314148,
-      "rewards/rejected": -0.4044874608516693,
-      "step": 50
-    },
-    {
-      "epoch": 2.5,
-      "grad_norm": 3.9862306118011475,
-      "learning_rate": 1.8055555555555557e-06,
-      "logits/chosen": -1.787440538406372,
-      "logits/rejected": -1.6402308940887451,
-      "logps/chosen": -483.3511657714844,
-      "logps/rejected": -451.0846252441406,
-      "loss": 0.2905,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.6469835042953491,
-      "rewards/margins": 1.1269721984863281,
-      "rewards/rejected": -0.4799886643886566,
-      "step": 60
-    },
-    {
-      "epoch": 2.9166666666666665,
-      "grad_norm": 3.802476406097412,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.8231675624847412,
-      "logits/rejected": -1.6182845830917358,
-      "logps/chosen": -482.34552001953125,
-      "logps/rejected": -476.08245849609375,
-      "loss": 0.2487,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7560733556747437,
-      "rewards/margins": 1.309217929840088,
-      "rewards/rejected": -0.553144633769989,
-      "step": 70
-    },
     {
       "epoch": 3.0,
-      "step": 72,
+      "step": 6,
       "total_flos": 0.0,
-      "train_loss": 0.43935056610239875,
-      "train_runtime": 212.525,
-      "train_samples_per_second": 3.317,
-      "train_steps_per_second": 0.339
+      "train_loss": 0.6918749014536539,
+      "train_runtime": 10.2523,
+      "train_samples_per_second": 2.341,
+      "train_steps_per_second": 0.585
     }
   ],
   "logging_steps": 10,
-  "max_steps": 72,
+  "max_steps": 6,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3651298d483387dd8e41a7f96f9b2df27290e8b94f31024d88b04779a9c64fdc
-size 6865
+oid sha256:50a9e1331ba9c68f2f41bad01edc382f75c5e0577212aa2083afc36604e5bfba
+size 6929