Initial private upload

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/merged_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/research_question_agent/checkpoint-669/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/research_question_agent/merged_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/DPO_configs.json

@@ -0,0 +1,181 @@
+{
+    "output_dir": "Trained_Models/Jackson0018/Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent",
+    "overwrite_output_dir": null,
+    "do_train": false,
+    "do_eval": false,
+    "do_predict": false,
+    "eval_strategy": "no",
+    "prediction_loss_only": false,
+    "per_device_train_batch_size": 1,
+    "per_device_eval_batch_size": 4,
+    "per_gpu_train_batch_size": null,
+    "per_gpu_eval_batch_size": null,
+    "gradient_accumulation_steps": 32,
+    "eval_accumulation_steps": 2,
+    "eval_delay": 0,
+    "torch_empty_cache_steps": 250,
+    "learning_rate": 1e-05,
+    "weight_decay": 0.01,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.999,
+    "adam_epsilon": 1e-08,
+    "max_grad_norm": 0.6,
+    "num_train_epochs": 3.0,
+    "max_steps": -1,
+    "lr_scheduler_type": "linear",
+    "lr_scheduler_kwargs": {},
+    "warmup_ratio": 0.1,
+    "warmup_steps": 0,
+    "log_level": "passive",
+    "log_level_replica": "warning",
+    "log_on_each_node": true,
+    "logging_dir": "Trained_Models/Jackson0018/Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/runs/Sep29_07-50-17_ai16",
+    "logging_strategy": "steps",
+    "logging_first_step": false,
+    "logging_steps": 10,
+    "logging_nan_inf_filter": false,
+    "save_strategy": "steps",
+    "save_steps": 10000,
+    "save_total_limit": null,
+    "save_safetensors": true,
+    "save_on_each_node": false,
+    "save_only_model": false,
+    "restore_callback_states_from_checkpoint": false,
+    "no_cuda": false,
+    "use_cpu": false,
+    "use_mps_device": false,
+    "seed": 3407,
+    "data_seed": 3407,
+    "jit_mode_eval": false,
+    "use_ipex": false,
+    "bf16": true,
+    "fp16": false,
+    "fp16_opt_level": "O1",
+    "half_precision_backend": "auto",
+    "bf16_full_eval": false,
+    "fp16_full_eval": false,
+    "tf32": null,
+    "local_rank": 0,
+    "ddp_backend": null,
+    "tpu_num_cores": null,
+    "tpu_metrics_debug": false,
+    "debug": [],
+    "dataloader_drop_last": false,
+    "eval_steps": null,
+    "dataloader_num_workers": 0,
+    "dataloader_prefetch_factor": null,
+    "past_index": -1,
+    "run_name": null,
+    "disable_tqdm": false,
+    "remove_unused_columns": true,
+    "label_names": null,
+    "load_best_model_at_end": false,
+    "metric_for_best_model": null,
+    "greater_is_better": null,
+    "ignore_data_skip": false,
+    "fsdp": [],
+    "fsdp_min_num_params": 0,
+    "fsdp_config": {
+        "min_num_params": 0,
+        "xla": false,
+        "xla_fsdp_v2": false,
+        "xla_fsdp_grad_ckpt": false
+    },
+    "fsdp_transformer_layer_cls_to_wrap": null,
+    "accelerator_config": {
+        "split_batches": false,
+        "dispatch_batches": null,
+        "even_batches": true,
+        "use_seedable_sampler": true,
+        "non_blocking": false,
+        "gradient_accumulation_kwargs": null
+    },
+    "deepspeed": null,
+    "label_smoothing_factor": 0.0,
+    "optim": "adamw_8bit",
+    "optim_args": null,
+    "adafactor": false,
+    "group_by_length": false,
+    "length_column_name": "length",
+    "report_to": [],
+    "ddp_find_unused_parameters": null,
+    "ddp_bucket_cap_mb": null,
+    "ddp_broadcast_buffers": null,
+    "dataloader_pin_memory": true,
+    "dataloader_persistent_workers": false,
+    "skip_memory_metrics": true,
+    "use_legacy_prediction_loop": false,
+    "push_to_hub": false,
+    "resume_from_checkpoint": null,
+    "hub_model_id": null,
+    "hub_strategy": "every_save",
+    "hub_token": "<HUB_TOKEN>",
+    "hub_private_repo": null,
+    "hub_always_push": false,
+    "hub_revision": null,
+    "gradient_checkpointing": false,
+    "gradient_checkpointing_kwargs": null,
+    "include_inputs_for_metrics": false,
+    "include_for_metrics": [],
+    "eval_do_concat_batches": true,
+    "fp16_backend": "auto",
+    "push_to_hub_model_id": null,
+    "push_to_hub_organization": null,
+    "push_to_hub_token": "<PUSH_TO_HUB_TOKEN>",
+    "mp_parameters": "",
+    "auto_find_batch_size": true,
+    "full_determinism": false,
+    "torchdynamo": null,
+    "ray_scope": "last",
+    "ddp_timeout": 1800,
+    "torch_compile": false,
+    "torch_compile_backend": null,
+    "torch_compile_mode": null,
+    "include_tokens_per_second": false,
+    "include_num_input_tokens_seen": false,
+    "neftune_noise_alpha": null,
+    "optim_target_modules": null,
+    "batch_eval_metrics": false,
+    "eval_on_start": false,
+    "use_liger_kernel": false,
+    "liger_kernel_config": null,
+    "eval_use_gather_object": false,
+    "average_tokens_across_devices": false,
+    "model_init_kwargs": null,
+    "ref_model_init_kwargs": null,
+    "model_adapter_name": null,
+    "ref_adapter_name": null,
+    "force_use_ref_model": false,
+    "disable_dropout": true,
+    "use_logits_to_keep": false,
+    "dataset_num_proc": 2,
+    "padding_value": null,
+    "label_pad_token_id": -100,
+    "max_prompt_length": 2000,
+    "max_completion_length": 2000,
+    "max_length": 4000,
+    "truncation_mode": "keep_end",
+    "padding_free": false,
+    "precompute_ref_log_probs": false,
+    "precompute_ref_batch_size": null,
+    "tools": null,
+    "loss_type": "sigmoid",
+    "use_liger_loss": false,
+    "base_model_attribute_name": "model",
+    "beta": 0.1,
+    "f_divergence_type": "reverse_kl",
+    "f_alpha_divergence_coef": 1.0,
+    "reference_free": false,
+    "label_smoothing": 0.0,
+    "use_weighting": false,
+    "rpo_alpha": null,
+    "ld_alpha": null,
+    "discopop_tau": 0.05,
+    "loss_weights": null,
+    "sync_ref_model": false,
+    "ref_model_mixup_alpha": 0.6,
+    "ref_model_sync_steps": 512,
+    "generate_during_eval": false,
+    "vllm_sampling_params": null,
+    "unsloth_num_chunks": -1
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/README.md

@@ -0,0 +1,70 @@
+---
+base_model: unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit
+library_name: transformers
+model_name: experiment_agent
+tags:
+- generated_from_trainer
+- trl
+- unsloth
+- dpo
+licence: license
+---
+
+# Model Card for experiment_agent
+
+This model is a fine-tuned version of [unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit](https://huggingface.co/unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jackson0530/ScientificPaperRetrieval_Train-Train_DPO_unsloth/runs/ye26t4kv) 
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.21.0
+- Transformers: 4.55.0
+- Pytorch: 2.7.1
+- Datasets: 3.6.0
+- Tokenizers: 0.21.4
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/README.md

@@ -0,0 +1,210 @@
+---
+base_model: unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit
+- dpo
+- lora
+- transformers
+- trl
+- unsloth
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.0

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "gate_proj",
+    "q_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d66bf1b56d3472a8100f4986a2eb02d1c146d1d4404d0bb16466e935f5b1e91
+size 479005064

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9566052a00f6f9197425ce3ab8234dc85bf7f8c5da8b84b6ba217b67ed425dc0
+size 243803397

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181c5f0270cf39930062ddfa3767a2481d0c360f120b11f8e25dbf533a1cdaba
+size 14645

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46728529080e683318d0592a9290e2f3e1dd31fa31190ac892162c71aa04cef
+size 1465

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/special_tokens_map.json

@@ -0,0 +1,25 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>"
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|im_end|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/trainer_state.json

@@ -0,0 +1,1024 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 669,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0449438202247191,
+      "grad_norm": 8.41971206665039,
+      "learning_rate": 1.3432835820895524e-06,
+      "logits/chosen": -0.5647975206375122,
+      "logits/rejected": -0.5565350651741028,
+      "logps/chosen": -1636.8466796875,
+      "logps/rejected": -1614.4744873046875,
+      "loss": 0.7095,
+      "rewards/accuracies": 0.3343749940395355,
+      "rewards/chosen": -0.010566463693976402,
+      "rewards/margins": -0.023785116150975227,
+      "rewards/rejected": 0.013218650594353676,
+      "step": 10
+    },
+    {
+      "epoch": 0.0898876404494382,
+      "grad_norm": 8.172579765319824,
+      "learning_rate": 2.835820895522388e-06,
+      "logits/chosen": -0.543634295463562,
+      "logits/rejected": -0.5405128002166748,
+      "logps/chosen": -1649.8961181640625,
+      "logps/rejected": -1622.677978515625,
+      "loss": 0.7067,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.004455108195543289,
+      "rewards/margins": -0.015273856930434704,
+      "rewards/rejected": 0.019728967919945717,
+      "step": 20
+    },
+    {
+      "epoch": 0.1348314606741573,
+      "grad_norm": 8.977705955505371,
+      "learning_rate": 4.3283582089552236e-06,
+      "logits/chosen": -0.5407521724700928,
+      "logits/rejected": -0.5417042374610901,
+      "logps/chosen": -1637.083251953125,
+      "logps/rejected": -1631.73193359375,
+      "loss": 0.6967,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.05982738733291626,
+      "rewards/margins": 0.003512363415211439,
+      "rewards/rejected": 0.05631502345204353,
+      "step": 30
+    },
+    {
+      "epoch": 0.1797752808988764,
+      "grad_norm": 12.203088760375977,
+      "learning_rate": 5.820895522388061e-06,
+      "logits/chosen": -0.5544255971908569,
+      "logits/rejected": -0.538857102394104,
+      "logps/chosen": -1644.1826171875,
+      "logps/rejected": -1592.37451171875,
+      "loss": 0.6957,
+      "rewards/accuracies": 0.49687498807907104,
+      "rewards/chosen": 0.15313825011253357,
+      "rewards/margins": 0.007030182983726263,
+      "rewards/rejected": 0.14610807597637177,
+      "step": 40
+    },
+    {
+      "epoch": 0.2247191011235955,
+      "grad_norm": 8.75937557220459,
+      "learning_rate": 7.313432835820896e-06,
+      "logits/chosen": -0.5435775518417358,
+      "logits/rejected": -0.550085186958313,
+      "logps/chosen": -1686.4521484375,
+      "logps/rejected": -1653.057861328125,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.3402329981327057,
+      "rewards/margins": 0.025076771154999733,
+      "rewards/rejected": 0.3151562511920929,
+      "step": 50
+    },
+    {
+      "epoch": 0.2696629213483146,
+      "grad_norm": 9.613944053649902,
+      "learning_rate": 8.805970149253732e-06,
+      "logits/chosen": -0.5430251359939575,
+      "logits/rejected": -0.5381388068199158,
+      "logps/chosen": -1612.47900390625,
+      "logps/rejected": -1611.5440673828125,
+      "loss": 0.7,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.4334062933921814,
+      "rewards/margins": 0.001930123195052147,
+      "rewards/rejected": 0.4314761757850647,
+      "step": 60
+    },
+    {
+      "epoch": 0.3146067415730337,
+      "grad_norm": 9.039813995361328,
+      "learning_rate": 9.966777408637874e-06,
+      "logits/chosen": -0.5394436717033386,
+      "logits/rejected": -0.5394075512886047,
+      "logps/chosen": -1656.6136474609375,
+      "logps/rejected": -1667.557861328125,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.4906249940395355,
+      "rewards/chosen": 0.371154248714447,
+      "rewards/margins": 0.0005123887094669044,
+      "rewards/rejected": 0.37064188718795776,
+      "step": 70
+    },
+    {
+      "epoch": 0.3595505617977528,
+      "grad_norm": 10.225497245788574,
+      "learning_rate": 9.800664451827243e-06,
+      "logits/chosen": -0.5414221882820129,
+      "logits/rejected": -0.5409548282623291,
+      "logps/chosen": -1660.9986572265625,
+      "logps/rejected": -1641.126708984375,
+      "loss": 0.6989,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.008593291975557804,
+      "rewards/margins": 0.009244749322533607,
+      "rewards/rejected": -0.017838040366768837,
+      "step": 80
+    },
+    {
+      "epoch": 0.4044943820224719,
+      "grad_norm": 7.43681001663208,
+      "learning_rate": 9.634551495016612e-06,
+      "logits/chosen": -0.5525733232498169,
+      "logits/rejected": -0.5447468757629395,
+      "logps/chosen": -1664.12890625,
+      "logps/rejected": -1647.4931640625,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -0.1387433558702469,
+      "rewards/margins": 0.023330822587013245,
+      "rewards/rejected": -0.16207417845726013,
+      "step": 90
+    },
+    {
+      "epoch": 0.449438202247191,
+      "grad_norm": 7.95203971862793,
+      "learning_rate": 9.468438538205981e-06,
+      "logits/chosen": -0.5224291086196899,
+      "logits/rejected": -0.5349761843681335,
+      "logps/chosen": -1696.936767578125,
+      "logps/rejected": -1658.69140625,
+      "loss": 0.7033,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.006925581488758326,
+      "rewards/margins": 0.005276698153465986,
+      "rewards/rejected": 0.001648884266614914,
+      "step": 100
+    },
+    {
+      "epoch": 0.4943820224719101,
+      "grad_norm": 8.228531837463379,
+      "learning_rate": 9.30232558139535e-06,
+      "logits/chosen": -0.5292581915855408,
+      "logits/rejected": -0.5189449191093445,
+      "logps/chosen": -1666.102294921875,
+      "logps/rejected": -1667.996826171875,
+      "loss": 0.7191,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.2481352537870407,
+      "rewards/margins": -0.02416202798485756,
+      "rewards/rejected": 0.27229729294776917,
+      "step": 110
+    },
+    {
+      "epoch": 0.5393258426966292,
+      "grad_norm": 8.233011245727539,
+      "learning_rate": 9.136212624584718e-06,
+      "logits/chosen": -0.5090035796165466,
+      "logits/rejected": -0.5002211928367615,
+      "logps/chosen": -1661.6923828125,
+      "logps/rejected": -1677.4013671875,
+      "loss": 0.7038,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.2764546573162079,
+      "rewards/margins": 0.007198885083198547,
+      "rewards/rejected": 0.26925572752952576,
+      "step": 120
+    },
+    {
+      "epoch": 0.5842696629213483,
+      "grad_norm": 8.092947006225586,
+      "learning_rate": 8.970099667774087e-06,
+      "logits/chosen": -0.4953341484069824,
+      "logits/rejected": -0.49804240465164185,
+      "logps/chosen": -1656.296875,
+      "logps/rejected": -1640.451171875,
+      "loss": 0.7029,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.20165541768074036,
+      "rewards/margins": 0.015356823801994324,
+      "rewards/rejected": 0.18629857897758484,
+      "step": 130
+    },
+    {
+      "epoch": 0.6292134831460674,
+      "grad_norm": 7.318077087402344,
+      "learning_rate": 8.803986710963456e-06,
+      "logits/chosen": -0.48578256368637085,
+      "logits/rejected": -0.48590534925460815,
+      "logps/chosen": -1637.2935791015625,
+      "logps/rejected": -1652.760498046875,
+      "loss": 0.7022,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.20414631068706512,
+      "rewards/margins": 0.011423548683524132,
+      "rewards/rejected": 0.19272276759147644,
+      "step": 140
+    },
+    {
+      "epoch": 0.6741573033707865,
+      "grad_norm": 7.625386714935303,
+      "learning_rate": 8.637873754152825e-06,
+      "logits/chosen": -0.5298885703086853,
+      "logits/rejected": -0.5132607221603394,
+      "logps/chosen": -1658.0570068359375,
+      "logps/rejected": -1633.949951171875,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.0400543212890625,
+      "rewards/margins": 0.04242260009050369,
+      "rewards/rejected": -0.0023682781029492617,
+      "step": 150
+    },
+    {
+      "epoch": 0.7191011235955056,
+      "grad_norm": 9.02912712097168,
+      "learning_rate": 8.471760797342193e-06,
+      "logits/chosen": -0.5048017501831055,
+      "logits/rejected": -0.5085188746452332,
+      "logps/chosen": -1651.9283447265625,
+      "logps/rejected": -1614.8092041015625,
+      "loss": 0.7017,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.15381185710430145,
+      "rewards/margins": 0.014438611455261707,
+      "rewards/rejected": 0.13937325775623322,
+      "step": 160
+    },
+    {
+      "epoch": 0.7640449438202247,
+      "grad_norm": 7.760510444641113,
+      "learning_rate": 8.305647840531562e-06,
+      "logits/chosen": -0.5189553499221802,
+      "logits/rejected": -0.5221869945526123,
+      "logps/chosen": -1663.1962890625,
+      "logps/rejected": -1626.8187255859375,
+      "loss": 0.696,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": 0.4035443663597107,
+      "rewards/margins": 0.027243101969361305,
+      "rewards/rejected": 0.37630128860473633,
+      "step": 170
+    },
+    {
+      "epoch": 0.8089887640449438,
+      "grad_norm": 7.571713924407959,
+      "learning_rate": 8.139534883720931e-06,
+      "logits/chosen": -0.5345529317855835,
+      "logits/rejected": -0.5297631025314331,
+      "logps/chosen": -1653.6646728515625,
+      "logps/rejected": -1661.974853515625,
+      "loss": 0.7057,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.721655547618866,
+      "rewards/margins": 0.01410127617418766,
+      "rewards/rejected": 0.7075542211532593,
+      "step": 180
+    },
+    {
+      "epoch": 0.8539325842696629,
+      "grad_norm": 7.918243408203125,
+      "learning_rate": 7.9734219269103e-06,
+      "logits/chosen": -0.5173817276954651,
+      "logits/rejected": -0.5203038454055786,
+      "logps/chosen": -1616.741455078125,
+      "logps/rejected": -1628.0203857421875,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 0.4363967776298523,
+      "rewards/margins": 0.03799329325556755,
+      "rewards/rejected": 0.39840349555015564,
+      "step": 190
+    },
+    {
+      "epoch": 0.898876404494382,
+      "grad_norm": 7.25588321685791,
+      "learning_rate": 7.807308970099668e-06,
+      "logits/chosen": -0.5121560096740723,
+      "logits/rejected": -0.5224347710609436,
+      "logps/chosen": -1623.5826416015625,
+      "logps/rejected": -1649.8167724609375,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.05758974701166153,
+      "rewards/margins": 0.0646430253982544,
+      "rewards/rejected": -0.007053279783576727,
+      "step": 200
+    },
+    {
+      "epoch": 0.9438202247191011,
+      "grad_norm": 7.8804097175598145,
+      "learning_rate": 7.641196013289037e-06,
+      "logits/chosen": -0.5074772238731384,
+      "logits/rejected": -0.5116940140724182,
+      "logps/chosen": -1675.5833740234375,
+      "logps/rejected": -1657.4105224609375,
+      "loss": 0.7086,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.3717197775840759,
+      "rewards/margins": 0.013349572196602821,
+      "rewards/rejected": -0.3850693702697754,
+      "step": 210
+    },
+    {
+      "epoch": 0.9887640449438202,
+      "grad_norm": 7.504465579986572,
+      "learning_rate": 7.475083056478406e-06,
+      "logits/chosen": -0.5111061334609985,
+      "logits/rejected": -0.5182801485061646,
+      "logps/chosen": -1713.6666259765625,
+      "logps/rejected": -1688.6156005859375,
+      "loss": 0.706,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": -0.3375091552734375,
+      "rewards/margins": 0.021358530968427658,
+      "rewards/rejected": -0.3588676452636719,
+      "step": 220
+    },
+    {
+      "epoch": 1.0314606741573034,
+      "grad_norm": 6.4764933586120605,
+      "learning_rate": 7.308970099667775e-06,
+      "logits/chosen": -0.5052555203437805,
+      "logits/rejected": -0.49862349033355713,
+      "logps/chosen": -1678.5240478515625,
+      "logps/rejected": -1671.0499267578125,
+      "loss": 0.5908,
+      "rewards/accuracies": 0.7434210777282715,
+      "rewards/chosen": 0.31448131799697876,
+      "rewards/margins": 0.27145013213157654,
+      "rewards/rejected": 0.04303119331598282,
+      "step": 230
+    },
+    {
+      "epoch": 1.0764044943820226,
+      "grad_norm": 6.172929763793945,
+      "learning_rate": 7.1428571428571436e-06,
+      "logits/chosen": -0.4970678389072418,
+      "logits/rejected": -0.4996468424797058,
+      "logps/chosen": -1620.894775390625,
+      "logps/rejected": -1650.440673828125,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": 0.7087951302528381,
+      "rewards/margins": 0.37826400995254517,
+      "rewards/rejected": 0.3305310606956482,
+      "step": 240
+    },
+    {
+      "epoch": 1.1213483146067416,
+      "grad_norm": 6.442656517028809,
+      "learning_rate": 6.976744186046513e-06,
+      "logits/chosen": -0.5012631416320801,
+      "logits/rejected": -0.4978242516517639,
+      "logps/chosen": -1661.751708984375,
+      "logps/rejected": -1665.3834228515625,
+      "loss": 0.5127,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": 0.6133396029472351,
+      "rewards/margins": 0.45191797614097595,
+      "rewards/rejected": 0.16142162680625916,
+      "step": 250
+    },
+    {
+      "epoch": 1.1662921348314608,
+      "grad_norm": 7.005346775054932,
+      "learning_rate": 6.810631229235881e-06,
+      "logits/chosen": -0.5041731595993042,
+      "logits/rejected": -0.520297646522522,
+      "logps/chosen": -1645.6292724609375,
+      "logps/rejected": -1593.59375,
+      "loss": 0.5327,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.367062509059906,
+      "rewards/margins": 0.40600281953811646,
+      "rewards/rejected": -0.038940299302339554,
+      "step": 260
+    },
+    {
+      "epoch": 1.2112359550561798,
+      "grad_norm": 6.600498199462891,
+      "learning_rate": 6.64451827242525e-06,
+      "logits/chosen": -0.504959225654602,
+      "logits/rejected": -0.50092613697052,
+      "logps/chosen": -1652.6773681640625,
+      "logps/rejected": -1632.398681640625,
+      "loss": 0.5225,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": 0.5157921314239502,
+      "rewards/margins": 0.43294817209243774,
+      "rewards/rejected": 0.08284398913383484,
+      "step": 270
+    },
+    {
+      "epoch": 1.256179775280899,
+      "grad_norm": 6.736950874328613,
+      "learning_rate": 6.4784053156146185e-06,
+      "logits/chosen": -0.5179052352905273,
+      "logits/rejected": -0.526940643787384,
+      "logps/chosen": -1627.15478515625,
+      "logps/rejected": -1616.917236328125,
+      "loss": 0.527,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.6045408844947815,
+      "rewards/margins": 0.4338344931602478,
+      "rewards/rejected": 0.1707063764333725,
+      "step": 280
+    },
+    {
+      "epoch": 1.301123595505618,
+      "grad_norm": 7.117891788482666,
+      "learning_rate": 6.312292358803988e-06,
+      "logits/chosen": -0.517137885093689,
+      "logits/rejected": -0.5145695209503174,
+      "logps/chosen": -1660.9166259765625,
+      "logps/rejected": -1675.412841796875,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.6492522358894348,
+      "rewards/margins": 0.45881325006484985,
+      "rewards/rejected": 0.19043894112110138,
+      "step": 290
+    },
+    {
+      "epoch": 1.346067415730337,
+      "grad_norm": 6.42925500869751,
+      "learning_rate": 6.146179401993356e-06,
+      "logits/chosen": -0.5063742399215698,
+      "logits/rejected": -0.5090619921684265,
+      "logps/chosen": -1695.0318603515625,
+      "logps/rejected": -1676.716552734375,
+      "loss": 0.484,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.5437876582145691,
+      "rewards/margins": 0.5404478311538696,
+      "rewards/rejected": 0.0033398643136024475,
+      "step": 300
+    },
+    {
+      "epoch": 1.3910112359550562,
+      "grad_norm": 6.849030017852783,
+      "learning_rate": 5.980066445182725e-06,
+      "logits/chosen": -0.500776469707489,
+      "logits/rejected": -0.4920951724052429,
+      "logps/chosen": -1684.324951171875,
+      "logps/rejected": -1695.865966796875,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.34679412841796875,
+      "rewards/margins": 0.5230782628059387,
+      "rewards/rejected": -0.1762840747833252,
+      "step": 310
+    },
+    {
+      "epoch": 1.4359550561797754,
+      "grad_norm": 6.4250969886779785,
+      "learning_rate": 5.8139534883720935e-06,
+      "logits/chosen": -0.4945620596408844,
+      "logits/rejected": -0.4994569718837738,
+      "logps/chosen": -1643.4566650390625,
+      "logps/rejected": -1635.079833984375,
+      "loss": 0.508,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": 0.28158336877822876,
+      "rewards/margins": 0.49037957191467285,
+      "rewards/rejected": -0.2087961882352829,
+      "step": 320
+    },
+    {
+      "epoch": 1.4808988764044944,
+      "grad_norm": 7.239423751831055,
+      "learning_rate": 5.647840531561463e-06,
+      "logits/chosen": -0.4839072823524475,
+      "logits/rejected": -0.4932027757167816,
+      "logps/chosen": -1683.430419921875,
+      "logps/rejected": -1663.859619140625,
+      "loss": 0.4708,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.5630651712417603,
+      "rewards/margins": 0.5979984402656555,
+      "rewards/rejected": -0.03493330255150795,
+      "step": 330
+    },
+    {
+      "epoch": 1.5258426966292133,
+      "grad_norm": 6.240829944610596,
+      "learning_rate": 5.481727574750831e-06,
+      "logits/chosen": -0.5244169235229492,
+      "logits/rejected": -0.5004735589027405,
+      "logps/chosen": -1670.196044921875,
+      "logps/rejected": -1666.2545166015625,
+      "loss": 0.502,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.4043809771537781,
+      "rewards/margins": 0.5101131200790405,
+      "rewards/rejected": -0.10573209822177887,
+      "step": 340
+    },
+    {
+      "epoch": 1.5707865168539326,
+      "grad_norm": 8.406665802001953,
+      "learning_rate": 5.3156146179402e-06,
+      "logits/chosen": -0.4983617663383484,
+      "logits/rejected": -0.5115659236907959,
+      "logps/chosen": -1620.291259765625,
+      "logps/rejected": -1653.3515625,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": 0.19796034693717957,
+      "rewards/margins": 0.5496314764022827,
+      "rewards/rejected": -0.35167109966278076,
+      "step": 350
+    },
+    {
+      "epoch": 1.6157303370786518,
+      "grad_norm": 8.227237701416016,
+      "learning_rate": 5.149501661129569e-06,
+      "logits/chosen": -0.5067149996757507,
+      "logits/rejected": -0.5047170519828796,
+      "logps/chosen": -1646.197998046875,
+      "logps/rejected": -1636.479736328125,
+      "loss": 0.4833,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.1900782585144043,
+      "rewards/margins": 0.5684794187545776,
+      "rewards/rejected": -0.37840116024017334,
+      "step": 360
+    },
+    {
+      "epoch": 1.6606741573033708,
+      "grad_norm": 6.621316432952881,
+      "learning_rate": 4.983388704318937e-06,
+      "logits/chosen": -0.5395005941390991,
+      "logits/rejected": -0.5313366055488586,
+      "logps/chosen": -1667.989501953125,
+      "logps/rejected": -1636.424560546875,
+      "loss": 0.4614,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.33250755071640015,
+      "rewards/margins": 0.6282721757888794,
+      "rewards/rejected": -0.29576462507247925,
+      "step": 370
+    },
+    {
+      "epoch": 1.7056179775280897,
+      "grad_norm": 7.328308582305908,
+      "learning_rate": 4.817275747508306e-06,
+      "logits/chosen": -0.5294896960258484,
+      "logits/rejected": -0.5275659561157227,
+      "logps/chosen": -1608.811279296875,
+      "logps/rejected": -1615.828857421875,
+      "loss": 0.4733,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.5715526342391968,
+      "rewards/margins": 0.6080284118652344,
+      "rewards/rejected": -0.036475833505392075,
+      "step": 380
+    },
+    {
+      "epoch": 1.750561797752809,
+      "grad_norm": 6.845993518829346,
+      "learning_rate": 4.651162790697675e-06,
+      "logits/chosen": -0.5189222097396851,
+      "logits/rejected": -0.5236440300941467,
+      "logps/chosen": -1629.564208984375,
+      "logps/rejected": -1620.7515869140625,
+      "loss": 0.4831,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 0.5099019408226013,
+      "rewards/margins": 0.5789185166358948,
+      "rewards/rejected": -0.06901657581329346,
+      "step": 390
+    },
+    {
+      "epoch": 1.7955056179775282,
+      "grad_norm": 6.671547889709473,
+      "learning_rate": 4.4850498338870435e-06,
+      "logits/chosen": -0.5234844088554382,
+      "logits/rejected": -0.5202063322067261,
+      "logps/chosen": -1641.2923583984375,
+      "logps/rejected": -1654.5943603515625,
+      "loss": 0.4701,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 0.37002792954444885,
+      "rewards/margins": 0.6203145980834961,
+      "rewards/rejected": -0.25028663873672485,
+      "step": 400
+    },
+    {
+      "epoch": 1.8404494382022472,
+      "grad_norm": 7.342043399810791,
+      "learning_rate": 4.318936877076413e-06,
+      "logits/chosen": -0.5412445068359375,
+      "logits/rejected": -0.5334212779998779,
+      "logps/chosen": -1657.1158447265625,
+      "logps/rejected": -1649.1107177734375,
+      "loss": 0.4836,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.34860578179359436,
+      "rewards/margins": 0.5773425698280334,
+      "rewards/rejected": -0.22873680293560028,
+      "step": 410
+    },
+    {
+      "epoch": 1.8853932584269661,
+      "grad_norm": 7.252129077911377,
+      "learning_rate": 4.152823920265781e-06,
+      "logits/chosen": -0.5296737551689148,
+      "logits/rejected": -0.5220682621002197,
+      "logps/chosen": -1667.012451171875,
+      "logps/rejected": -1626.130615234375,
+      "loss": 0.478,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.5405913591384888,
+      "rewards/margins": 0.5873361825942993,
+      "rewards/rejected": -0.04674474522471428,
+      "step": 420
+    },
+    {
+      "epoch": 1.9303370786516854,
+      "grad_norm": 7.426218509674072,
+      "learning_rate": 3.98671096345515e-06,
+      "logits/chosen": -0.5249942541122437,
+      "logits/rejected": -0.5246230959892273,
+      "logps/chosen": -1684.589599609375,
+      "logps/rejected": -1637.92041015625,
+      "loss": 0.4955,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.6054187417030334,
+      "rewards/margins": 0.5561720132827759,
+      "rewards/rejected": 0.049246758222579956,
+      "step": 430
+    },
+    {
+      "epoch": 1.9752808988764046,
+      "grad_norm": 6.802650451660156,
+      "learning_rate": 3.8205980066445185e-06,
+      "logits/chosen": -0.5376031398773193,
+      "logits/rejected": -0.5206517577171326,
+      "logps/chosen": -1635.8851318359375,
+      "logps/rejected": -1639.7294921875,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.833863377571106,
+      "rewards/margins": 0.5611246228218079,
+      "rewards/rejected": 0.2727387845516205,
+      "step": 440
+    },
+    {
+      "epoch": 2.0179775280898875,
+      "grad_norm": 5.3229265213012695,
+      "learning_rate": 3.6544850498338876e-06,
+      "logits/chosen": -0.5227991938591003,
+      "logits/rejected": -0.526660144329071,
+      "logps/chosen": -1625.6988525390625,
+      "logps/rejected": -1614.040771484375,
+      "loss": 0.414,
+      "rewards/accuracies": 0.9111841917037964,
+      "rewards/chosen": 0.9086302518844604,
+      "rewards/margins": 0.7858190536499023,
+      "rewards/rejected": 0.12281119078397751,
+      "step": 450
+    },
+    {
+      "epoch": 2.0629213483146067,
+      "grad_norm": 5.2561516761779785,
+      "learning_rate": 3.4883720930232564e-06,
+      "logits/chosen": -0.538662314414978,
+      "logits/rejected": -0.5180272459983826,
+      "logps/chosen": -1612.210693359375,
+      "logps/rejected": -1614.930908203125,
+      "loss": 0.3374,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 0.8260666131973267,
+      "rewards/margins": 1.0068495273590088,
+      "rewards/rejected": -0.1807830035686493,
+      "step": 460
+    },
+    {
+      "epoch": 2.107865168539326,
+      "grad_norm": 5.642768859863281,
+      "learning_rate": 3.322259136212625e-06,
+      "logits/chosen": -0.5233631730079651,
+      "logits/rejected": -0.5245693922042847,
+      "logps/chosen": -1631.31689453125,
+      "logps/rejected": -1639.3062744140625,
+      "loss": 0.3276,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.6572339534759521,
+      "rewards/margins": 1.0290508270263672,
+      "rewards/rejected": -0.3718169629573822,
+      "step": 470
+    },
+    {
+      "epoch": 2.152808988764045,
+      "grad_norm": 5.401882171630859,
+      "learning_rate": 3.156146179401994e-06,
+      "logits/chosen": -0.538943350315094,
+      "logits/rejected": -0.5362802743911743,
+      "logps/chosen": -1624.647705078125,
+      "logps/rejected": -1646.9886474609375,
+      "loss": 0.3255,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 0.38137590885162354,
+      "rewards/margins": 1.0469489097595215,
+      "rewards/rejected": -0.665573000907898,
+      "step": 480
+    },
+    {
+      "epoch": 2.197752808988764,
+      "grad_norm": 5.061157703399658,
+      "learning_rate": 2.9900332225913626e-06,
+      "logits/chosen": -0.534186840057373,
+      "logits/rejected": -0.5299129486083984,
+      "logps/chosen": -1611.435546875,
+      "logps/rejected": -1608.452880859375,
+      "loss": 0.335,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": 0.262370765209198,
+      "rewards/margins": 1.0311752557754517,
+      "rewards/rejected": -0.7688044309616089,
+      "step": 490
+    },
+    {
+      "epoch": 2.242696629213483,
+      "grad_norm": 5.667867660522461,
+      "learning_rate": 2.8239202657807313e-06,
+      "logits/chosen": -0.5238770842552185,
+      "logits/rejected": -0.5233170390129089,
+      "logps/chosen": -1681.3297119140625,
+      "logps/rejected": -1670.050048828125,
+      "loss": 0.3283,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.6769800782203674,
+      "rewards/margins": 1.0591976642608643,
+      "rewards/rejected": -0.382217675447464,
+      "step": 500
+    },
+    {
+      "epoch": 2.2876404494382023,
+      "grad_norm": 5.56412410736084,
+      "learning_rate": 2.6578073089701e-06,
+      "logits/chosen": -0.5160372853279114,
+      "logits/rejected": -0.5247339010238647,
+      "logps/chosen": -1672.7626953125,
+      "logps/rejected": -1685.137451171875,
+      "loss": 0.3162,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.8809404373168945,
+      "rewards/margins": 1.111767053604126,
+      "rewards/rejected": -0.23082669079303741,
+      "step": 510
+    },
+    {
+      "epoch": 2.3325842696629215,
+      "grad_norm": 5.156087875366211,
+      "learning_rate": 2.4916943521594684e-06,
+      "logits/chosen": -0.5284621715545654,
+      "logits/rejected": -0.5034042596817017,
+      "logps/chosen": -1669.8564453125,
+      "logps/rejected": -1671.060302734375,
+      "loss": 0.314,
+      "rewards/accuracies": 0.965624988079071,
+      "rewards/chosen": 0.7527168989181519,
+      "rewards/margins": 1.123870611190796,
+      "rewards/rejected": -0.37115368247032166,
+      "step": 520
+    },
+    {
+      "epoch": 2.3775280898876403,
+      "grad_norm": 5.17440938949585,
+      "learning_rate": 2.3255813953488376e-06,
+      "logits/chosen": -0.5137378573417664,
+      "logits/rejected": -0.5173753499984741,
+      "logps/chosen": -1650.34375,
+      "logps/rejected": -1654.510986328125,
+      "loss": 0.3163,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 0.476835161447525,
+      "rewards/margins": 1.0856043100357056,
+      "rewards/rejected": -0.6087690591812134,
+      "step": 530
+    },
+    {
+      "epoch": 2.4224719101123595,
+      "grad_norm": 6.027777194976807,
+      "learning_rate": 2.1594684385382063e-06,
+      "logits/chosen": -0.5290040969848633,
+      "logits/rejected": -0.5300403833389282,
+      "logps/chosen": -1685.980712890625,
+      "logps/rejected": -1669.431884765625,
+      "loss": 0.3284,
+      "rewards/accuracies": 0.965624988079071,
+      "rewards/chosen": 0.3279837369918823,
+      "rewards/margins": 1.065712332725525,
+      "rewards/rejected": -0.737728476524353,
+      "step": 540
+    },
+    {
+      "epoch": 2.4674157303370787,
+      "grad_norm": 5.797597408294678,
+      "learning_rate": 1.993355481727575e-06,
+      "logits/chosen": -0.5459321737289429,
+      "logits/rejected": -0.5469938516616821,
+      "logps/chosen": -1586.365234375,
+      "logps/rejected": -1629.16943359375,
+      "loss": 0.3377,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.41557711362838745,
+      "rewards/margins": 1.04073965549469,
+      "rewards/rejected": -0.6251626014709473,
+      "step": 550
+    },
+    {
+      "epoch": 2.512359550561798,
+      "grad_norm": 5.85587215423584,
+      "learning_rate": 1.8272425249169438e-06,
+      "logits/chosen": -0.5439457297325134,
+      "logits/rejected": -0.5378574728965759,
+      "logps/chosen": -1665.3310546875,
+      "logps/rejected": -1670.1988525390625,
+      "loss": 0.3114,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.520895779132843,
+      "rewards/margins": 1.1236566305160522,
+      "rewards/rejected": -0.6027609705924988,
+      "step": 560
+    },
+    {
+      "epoch": 2.5573033707865167,
+      "grad_norm": 5.2037458419799805,
+      "learning_rate": 1.6611295681063126e-06,
+      "logits/chosen": -0.5538147687911987,
+      "logits/rejected": -0.5644603371620178,
+      "logps/chosen": -1645.1334228515625,
+      "logps/rejected": -1638.0411376953125,
+      "loss": 0.3055,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5205073356628418,
+      "rewards/margins": 1.148437738418579,
+      "rewards/rejected": -0.6279304027557373,
+      "step": 570
+    },
+    {
+      "epoch": 2.602247191011236,
+      "grad_norm": 5.944153785705566,
+      "learning_rate": 1.4950166112956813e-06,
+      "logits/chosen": -0.5424203872680664,
+      "logits/rejected": -0.5553634762763977,
+      "logps/chosen": -1630.100341796875,
+      "logps/rejected": -1639.7135009765625,
+      "loss": 0.3289,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.46930861473083496,
+      "rewards/margins": 1.0725914239883423,
+      "rewards/rejected": -0.6032828092575073,
+      "step": 580
+    },
+    {
+      "epoch": 2.647191011235955,
+      "grad_norm": 5.082998752593994,
+      "learning_rate": 1.32890365448505e-06,
+      "logits/chosen": -0.5557939410209656,
+      "logits/rejected": -0.5634464025497437,
+      "logps/chosen": -1674.44921875,
+      "logps/rejected": -1665.8763427734375,
+      "loss": 0.311,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.5268303751945496,
+      "rewards/margins": 1.155867099761963,
+      "rewards/rejected": -0.6290367841720581,
+      "step": 590
+    },
+    {
+      "epoch": 2.692134831460674,
+      "grad_norm": 6.137351989746094,
+      "learning_rate": 1.1627906976744188e-06,
+      "logits/chosen": -0.5621194243431091,
+      "logits/rejected": -0.5413273572921753,
+      "logps/chosen": -1648.4447021484375,
+      "logps/rejected": -1647.3736572265625,
+      "loss": 0.3104,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.5918968319892883,
+      "rewards/margins": 1.1398565769195557,
+      "rewards/rejected": -0.5479596853256226,
+      "step": 600
+    },
+    {
+      "epoch": 2.737078651685393,
+      "grad_norm": 5.4996490478515625,
+      "learning_rate": 9.966777408637875e-07,
+      "logits/chosen": -0.5511677265167236,
+      "logits/rejected": -0.5449541211128235,
+      "logps/chosen": -1658.724609375,
+      "logps/rejected": -1671.2269287109375,
+      "loss": 0.3269,
+      "rewards/accuracies": 0.9593750238418579,
+      "rewards/chosen": 0.5969556570053101,
+      "rewards/margins": 1.071141242980957,
+      "rewards/rejected": -0.4741855561733246,
+      "step": 610
+    },
+    {
+      "epoch": 2.7820224719101123,
+      "grad_norm": 7.489591598510742,
+      "learning_rate": 8.305647840531563e-07,
+      "logits/chosen": -0.5342198610305786,
+      "logits/rejected": -0.5351474285125732,
+      "logps/chosen": -1673.1702880859375,
+      "logps/rejected": -1652.4296875,
+      "loss": 0.3087,
+      "rewards/accuracies": 0.9593750238418579,
+      "rewards/chosen": 0.5696877241134644,
+      "rewards/margins": 1.1663155555725098,
+      "rewards/rejected": -0.5966278314590454,
+      "step": 620
+    },
+    {
+      "epoch": 2.8269662921348315,
+      "grad_norm": 5.481851100921631,
+      "learning_rate": 6.64451827242525e-07,
+      "logits/chosen": -0.5519033670425415,
+      "logits/rejected": -0.5528594851493835,
+      "logps/chosen": -1630.025634765625,
+      "logps/rejected": -1657.2965087890625,
+      "loss": 0.3035,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.5027799606323242,
+      "rewards/margins": 1.1599414348602295,
+      "rewards/rejected": -0.6571615934371948,
+      "step": 630
+    },
+    {
+      "epoch": 2.8719101123595507,
+      "grad_norm": 6.2103376388549805,
+      "learning_rate": 4.983388704318938e-07,
+      "logits/chosen": -0.5550025701522827,
+      "logits/rejected": -0.5428484678268433,
+      "logps/chosen": -1660.6510009765625,
+      "logps/rejected": -1650.1458740234375,
+      "loss": 0.3054,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 0.48065298795700073,
+      "rewards/margins": 1.1632822751998901,
+      "rewards/rejected": -0.6826292872428894,
+      "step": 640
+    },
+    {
+      "epoch": 2.9168539325842695,
+      "grad_norm": 5.274155616760254,
+      "learning_rate": 3.322259136212625e-07,
+      "logits/chosen": -0.559792160987854,
+      "logits/rejected": -0.5550666451454163,
+      "logps/chosen": -1683.5833740234375,
+      "logps/rejected": -1644.792236328125,
+      "loss": 0.2943,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": 0.483023077249527,
+      "rewards/margins": 1.2071751356124878,
+      "rewards/rejected": -0.7241520285606384,
+      "step": 650
+    },
+    {
+      "epoch": 2.9617977528089887,
+      "grad_norm": 5.632884502410889,
+      "learning_rate": 1.6611295681063126e-07,
+      "logits/chosen": -0.5550089478492737,
+      "logits/rejected": -0.5735629796981812,
+      "logps/chosen": -1673.248779296875,
+      "logps/rejected": -1636.8924560546875,
+      "loss": 0.3071,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": 0.47631826996803284,
+      "rewards/margins": 1.1436049938201904,
+      "rewards/rejected": -0.6672865748405457,
+      "step": 660
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 669,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/checkpoint-669/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea74b09aba8751ccf31169f860970b324a96335b8e52ce8b9b0699d68927693d
+size 7057

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.55.0"
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e3cbf78021d13f0b4ae980ae979987f39d5f3bdfe608e70c702f0c140af2ed
+size 4982131536

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be573dbdc2126e1c398f22a3eb2d1582f41c1190b4f39f83e0978cf9c7147c6b
+size 4932949336

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15cae31cc6b5ea2ed26deb3ddfc2991fe49ff3a7b9e390ace9c0e682daef5c2b
+size 2428723160

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/model.safetensors.index.json

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_parameters": 3085938688,
+    "total_size": 12343754752
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/merged_model/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/model_args.json

@@ -0,0 +1,4 @@
+{
+    "model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+    "agent_name": "experiment_agent"
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/experiment_agent/train_args.json

@@ -0,0 +1,3 @@
+{
+    "load_in_4bit": true
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/DPO_configs.json

@@ -0,0 +1,181 @@
+{
+    "output_dir": "Trained_Models/Jackson0018/Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent",
+    "overwrite_output_dir": null,
+    "do_train": false,
+    "do_eval": false,
+    "do_predict": false,
+    "eval_strategy": "no",
+    "prediction_loss_only": false,
+    "per_device_train_batch_size": 1,
+    "per_device_eval_batch_size": 4,
+    "per_gpu_train_batch_size": null,
+    "per_gpu_eval_batch_size": null,
+    "gradient_accumulation_steps": 32,
+    "eval_accumulation_steps": 2,
+    "eval_delay": 0,
+    "torch_empty_cache_steps": 250,
+    "learning_rate": 1e-05,
+    "weight_decay": 0.01,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.999,
+    "adam_epsilon": 1e-08,
+    "max_grad_norm": 0.6,
+    "num_train_epochs": 3.0,
+    "max_steps": -1,
+    "lr_scheduler_type": "linear",
+    "lr_scheduler_kwargs": {},
+    "warmup_ratio": 0.1,
+    "warmup_steps": 0,
+    "log_level": "passive",
+    "log_level_replica": "warning",
+    "log_on_each_node": true,
+    "logging_dir": "Trained_Models/Jackson0018/Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/runs/Sep28_23-51-45_ai21",
+    "logging_strategy": "steps",
+    "logging_first_step": false,
+    "logging_steps": 10,
+    "logging_nan_inf_filter": false,
+    "save_strategy": "steps",
+    "save_steps": 10000,
+    "save_total_limit": null,
+    "save_safetensors": true,
+    "save_on_each_node": false,
+    "save_only_model": false,
+    "restore_callback_states_from_checkpoint": false,
+    "no_cuda": false,
+    "use_cpu": false,
+    "use_mps_device": false,
+    "seed": 3407,
+    "data_seed": 3407,
+    "jit_mode_eval": false,
+    "use_ipex": false,
+    "bf16": true,
+    "fp16": false,
+    "fp16_opt_level": "O1",
+    "half_precision_backend": "auto",
+    "bf16_full_eval": false,
+    "fp16_full_eval": false,
+    "tf32": null,
+    "local_rank": 0,
+    "ddp_backend": null,
+    "tpu_num_cores": null,
+    "tpu_metrics_debug": false,
+    "debug": [],
+    "dataloader_drop_last": false,
+    "eval_steps": null,
+    "dataloader_num_workers": 0,
+    "dataloader_prefetch_factor": null,
+    "past_index": -1,
+    "run_name": null,
+    "disable_tqdm": false,
+    "remove_unused_columns": true,
+    "label_names": null,
+    "load_best_model_at_end": false,
+    "metric_for_best_model": null,
+    "greater_is_better": null,
+    "ignore_data_skip": false,
+    "fsdp": [],
+    "fsdp_min_num_params": 0,
+    "fsdp_config": {
+        "min_num_params": 0,
+        "xla": false,
+        "xla_fsdp_v2": false,
+        "xla_fsdp_grad_ckpt": false
+    },
+    "fsdp_transformer_layer_cls_to_wrap": null,
+    "accelerator_config": {
+        "split_batches": false,
+        "dispatch_batches": null,
+        "even_batches": true,
+        "use_seedable_sampler": true,
+        "non_blocking": false,
+        "gradient_accumulation_kwargs": null
+    },
+    "deepspeed": null,
+    "label_smoothing_factor": 0.0,
+    "optim": "adamw_8bit",
+    "optim_args": null,
+    "adafactor": false,
+    "group_by_length": false,
+    "length_column_name": "length",
+    "report_to": [],
+    "ddp_find_unused_parameters": null,
+    "ddp_bucket_cap_mb": null,
+    "ddp_broadcast_buffers": null,
+    "dataloader_pin_memory": true,
+    "dataloader_persistent_workers": false,
+    "skip_memory_metrics": true,
+    "use_legacy_prediction_loop": false,
+    "push_to_hub": false,
+    "resume_from_checkpoint": null,
+    "hub_model_id": null,
+    "hub_strategy": "every_save",
+    "hub_token": "<HUB_TOKEN>",
+    "hub_private_repo": null,
+    "hub_always_push": false,
+    "hub_revision": null,
+    "gradient_checkpointing": false,
+    "gradient_checkpointing_kwargs": null,
+    "include_inputs_for_metrics": false,
+    "include_for_metrics": [],
+    "eval_do_concat_batches": true,
+    "fp16_backend": "auto",
+    "push_to_hub_model_id": null,
+    "push_to_hub_organization": null,
+    "push_to_hub_token": "<PUSH_TO_HUB_TOKEN>",
+    "mp_parameters": "",
+    "auto_find_batch_size": true,
+    "full_determinism": false,
+    "torchdynamo": null,
+    "ray_scope": "last",
+    "ddp_timeout": 1800,
+    "torch_compile": false,
+    "torch_compile_backend": null,
+    "torch_compile_mode": null,
+    "include_tokens_per_second": false,
+    "include_num_input_tokens_seen": false,
+    "neftune_noise_alpha": null,
+    "optim_target_modules": null,
+    "batch_eval_metrics": false,
+    "eval_on_start": false,
+    "use_liger_kernel": false,
+    "liger_kernel_config": null,
+    "eval_use_gather_object": false,
+    "average_tokens_across_devices": false,
+    "model_init_kwargs": null,
+    "ref_model_init_kwargs": null,
+    "model_adapter_name": null,
+    "ref_adapter_name": null,
+    "force_use_ref_model": false,
+    "disable_dropout": true,
+    "use_logits_to_keep": false,
+    "dataset_num_proc": 2,
+    "padding_value": null,
+    "label_pad_token_id": -100,
+    "max_prompt_length": 2000,
+    "max_completion_length": 2000,
+    "max_length": 4000,
+    "truncation_mode": "keep_end",
+    "padding_free": false,
+    "precompute_ref_log_probs": false,
+    "precompute_ref_batch_size": null,
+    "tools": null,
+    "loss_type": "sigmoid",
+    "use_liger_loss": false,
+    "base_model_attribute_name": "model",
+    "beta": 0.1,
+    "f_divergence_type": "reverse_kl",
+    "f_alpha_divergence_coef": 1.0,
+    "reference_free": false,
+    "label_smoothing": 0.0,
+    "use_weighting": false,
+    "rpo_alpha": null,
+    "ld_alpha": null,
+    "discopop_tau": 0.05,
+    "loss_weights": null,
+    "sync_ref_model": false,
+    "ref_model_mixup_alpha": 0.6,
+    "ref_model_sync_steps": 512,
+    "generate_during_eval": false,
+    "vllm_sampling_params": null,
+    "unsloth_num_chunks": -1
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/README.md

@@ -0,0 +1,70 @@
+---
+base_model: unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit
+library_name: transformers
+model_name: method_agent
+tags:
+- generated_from_trainer
+- unsloth
+- trl
+- dpo
+licence: license
+---
+
+# Model Card for method_agent
+
+This model is a fine-tuned version of [unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit](https://huggingface.co/unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jackson0530/ScientificPaperRetrieval_Train-Train_DPO_unsloth/runs/nd2l9u6s) 
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.21.0
+- Transformers: 4.55.0
+- Pytorch: 2.7.1
+- Datasets: 3.6.0
+- Tokenizers: 0.21.4
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/README.md

@@ -0,0 +1,210 @@
+---
+base_model: unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit
+- dpo
+- lora
+- transformers
+- trl
+- unsloth
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.0

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "gate_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj",
+    "q_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ae24012a3ea8f7fce9a008de9f97181e12bbbaf0531c68a72541ea472974179
+size 479005064

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:049d3edc908f4459bc1140a73475a4423172563d986ff594d44623c8f1032f04
+size 243803397

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181c5f0270cf39930062ddfa3767a2481d0c360f120b11f8e25dbf533a1cdaba
+size 14645

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46728529080e683318d0592a9290e2f3e1dd31fa31190ac892162c71aa04cef
+size 1465

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/special_tokens_map.json

@@ -0,0 +1,25 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>"
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|im_end|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/trainer_state.json

@@ -0,0 +1,1024 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 669,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0449438202247191,
+      "grad_norm": 7.2400288581848145,
+      "learning_rate": 1.3432835820895524e-06,
+      "logits/chosen": -0.6259505748748779,
+      "logits/rejected": -0.624777615070343,
+      "logps/chosen": -1442.914306640625,
+      "logps/rejected": -1479.676025390625,
+      "loss": 0.6947,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": 0.0029697995632886887,
+      "rewards/margins": 0.0050926790572702885,
+      "rewards/rejected": -0.002122878096997738,
+      "step": 10
+    },
+    {
+      "epoch": 0.0898876404494382,
+      "grad_norm": 7.600191593170166,
+      "learning_rate": 2.835820895522388e-06,
+      "logits/chosen": -0.6153338551521301,
+      "logits/rejected": -0.6100883483886719,
+      "logps/chosen": -1482.649658203125,
+      "logps/rejected": -1481.495849609375,
+      "loss": 0.7043,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.029805928468704224,
+      "rewards/margins": -0.011171265505254269,
+      "rewards/rejected": -0.01863466389477253,
+      "step": 20
+    },
+    {
+      "epoch": 0.1348314606741573,
+      "grad_norm": 7.326368808746338,
+      "learning_rate": 4.3283582089552236e-06,
+      "logits/chosen": -0.6203653216362,
+      "logits/rejected": -0.6188796162605286,
+      "logps/chosen": -1467.7874755859375,
+      "logps/rejected": -1467.8643798828125,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.044243909418582916,
+      "rewards/margins": 0.018139898777008057,
+      "rewards/rejected": -0.062383800745010376,
+      "step": 30
+    },
+    {
+      "epoch": 0.1797752808988764,
+      "grad_norm": 7.233694553375244,
+      "learning_rate": 5.820895522388061e-06,
+      "logits/chosen": -0.603333592414856,
+      "logits/rejected": -0.6177260279655457,
+      "logps/chosen": -1480.260498046875,
+      "logps/rejected": -1457.1766357421875,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.09666772186756134,
+      "rewards/margins": 0.009738157503306866,
+      "rewards/rejected": -0.10640586912631989,
+      "step": 40
+    },
+    {
+      "epoch": 0.2247191011235955,
+      "grad_norm": 8.360169410705566,
+      "learning_rate": 7.313432835820896e-06,
+      "logits/chosen": -0.6346372961997986,
+      "logits/rejected": -0.6143137812614441,
+      "logps/chosen": -1467.9993896484375,
+      "logps/rejected": -1486.096923828125,
+      "loss": 0.6953,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.1760435253381729,
+      "rewards/margins": 0.0075997160747647285,
+      "rewards/rejected": -0.18364325165748596,
+      "step": 50
+    },
+    {
+      "epoch": 0.2696629213483146,
+      "grad_norm": 7.651188850402832,
+      "learning_rate": 8.805970149253732e-06,
+      "logits/chosen": -0.6099546551704407,
+      "logits/rejected": -0.603184700012207,
+      "logps/chosen": -1467.3143310546875,
+      "logps/rejected": -1464.572509765625,
+      "loss": 0.7023,
+      "rewards/accuracies": 0.47187501192092896,
+      "rewards/chosen": -0.21066781878471375,
+      "rewards/margins": -0.0032155998051166534,
+      "rewards/rejected": -0.2074522078037262,
+      "step": 60
+    },
+    {
+      "epoch": 0.3146067415730337,
+      "grad_norm": 7.600541114807129,
+      "learning_rate": 9.966777408637874e-06,
+      "logits/chosen": -0.6103735566139221,
+      "logits/rejected": -0.6086291074752808,
+      "logps/chosen": -1429.90673828125,
+      "logps/rejected": -1434.4637451171875,
+      "loss": 0.6975,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": -0.12626130878925323,
+      "rewards/margins": 0.006277655251324177,
+      "rewards/rejected": -0.13253895938396454,
+      "step": 70
+    },
+    {
+      "epoch": 0.3595505617977528,
+      "grad_norm": 7.452111721038818,
+      "learning_rate": 9.800664451827243e-06,
+      "logits/chosen": -0.6045969724655151,
+      "logits/rejected": -0.6059508919715881,
+      "logps/chosen": -1478.626953125,
+      "logps/rejected": -1493.902587890625,
+      "loss": 0.7037,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": -0.20764172077178955,
+      "rewards/margins": -0.005739938467741013,
+      "rewards/rejected": -0.20190176367759705,
+      "step": 80
+    },
+    {
+      "epoch": 0.4044943820224719,
+      "grad_norm": 7.821183681488037,
+      "learning_rate": 9.634551495016612e-06,
+      "logits/chosen": -0.624080240726471,
+      "logits/rejected": -0.6179949045181274,
+      "logps/chosen": -1482.8621826171875,
+      "logps/rejected": -1521.285888671875,
+      "loss": 0.6984,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5030026435852051,
+      "rewards/margins": 0.013752209953963757,
+      "rewards/rejected": -0.5167548656463623,
+      "step": 90
+    },
+    {
+      "epoch": 0.449438202247191,
+      "grad_norm": 7.3397536277771,
+      "learning_rate": 9.468438538205981e-06,
+      "logits/chosen": -0.6049561500549316,
+      "logits/rejected": -0.593718945980072,
+      "logps/chosen": -1475.85498046875,
+      "logps/rejected": -1476.67431640625,
+      "loss": 0.6979,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.6247831583023071,
+      "rewards/margins": 0.01662410795688629,
+      "rewards/rejected": -0.641407310962677,
+      "step": 100
+    },
+    {
+      "epoch": 0.4943820224719101,
+      "grad_norm": 7.887584686279297,
+      "learning_rate": 9.30232558139535e-06,
+      "logits/chosen": -0.6068257093429565,
+      "logits/rejected": -0.5953084826469421,
+      "logps/chosen": -1477.0902099609375,
+      "logps/rejected": -1462.1424560546875,
+      "loss": 0.6949,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.7456911206245422,
+      "rewards/margins": 0.02874937281012535,
+      "rewards/rejected": -0.7744405269622803,
+      "step": 110
+    },
+    {
+      "epoch": 0.5393258426966292,
+      "grad_norm": 7.936194896697998,
+      "learning_rate": 9.136212624584718e-06,
+      "logits/chosen": -0.6176148653030396,
+      "logits/rejected": -0.6264122128486633,
+      "logps/chosen": -1426.8101806640625,
+      "logps/rejected": -1441.458251953125,
+      "loss": 0.6985,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.7066472768783569,
+      "rewards/margins": 0.020474322140216827,
+      "rewards/rejected": -0.7271216511726379,
+      "step": 120
+    },
+    {
+      "epoch": 0.5842696629213483,
+      "grad_norm": 7.374553680419922,
+      "learning_rate": 8.970099667774087e-06,
+      "logits/chosen": -0.6316522359848022,
+      "logits/rejected": -0.612767219543457,
+      "logps/chosen": -1424.370849609375,
+      "logps/rejected": -1446.091552734375,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5253143310546875,
+      "rewards/margins": 0.03358197957277298,
+      "rewards/rejected": -0.5588963031768799,
+      "step": 130
+    },
+    {
+      "epoch": 0.6292134831460674,
+      "grad_norm": 7.252725124359131,
+      "learning_rate": 8.803986710963456e-06,
+      "logits/chosen": -0.5971549153327942,
+      "logits/rejected": -0.6010578870773315,
+      "logps/chosen": -1466.6973876953125,
+      "logps/rejected": -1467.979248046875,
+      "loss": 0.7016,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -0.5401462316513062,
+      "rewards/margins": 0.01154586486518383,
+      "rewards/rejected": -0.5516921281814575,
+      "step": 140
+    },
+    {
+      "epoch": 0.6741573033707865,
+      "grad_norm": 7.468507289886475,
+      "learning_rate": 8.637873754152825e-06,
+      "logits/chosen": -0.6183695793151855,
+      "logits/rejected": -0.6090121865272522,
+      "logps/chosen": -1442.530517578125,
+      "logps/rejected": -1445.8988037109375,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.48295894265174866,
+      "rewards/margins": 0.03915649652481079,
+      "rewards/rejected": -0.5221154093742371,
+      "step": 150
+    },
+    {
+      "epoch": 0.7191011235955056,
+      "grad_norm": 11.664162635803223,
+      "learning_rate": 8.471760797342193e-06,
+      "logits/chosen": -0.5982354283332825,
+      "logits/rejected": -0.5825232863426208,
+      "logps/chosen": -1513.6202392578125,
+      "logps/rejected": -1509.6234130859375,
+      "loss": 0.7117,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": -0.4665093421936035,
+      "rewards/margins": -0.006485844496637583,
+      "rewards/rejected": -0.4600234627723694,
+      "step": 160
+    },
+    {
+      "epoch": 0.7640449438202247,
+      "grad_norm": 8.19553279876709,
+      "learning_rate": 8.305647840531562e-06,
+      "logits/chosen": -0.6178793907165527,
+      "logits/rejected": -0.6187315583229065,
+      "logps/chosen": -1486.3048095703125,
+      "logps/rejected": -1474.712890625,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.09939844161272049,
+      "rewards/margins": 0.030898302793502808,
+      "rewards/rejected": -0.1302967518568039,
+      "step": 170
+    },
+    {
+      "epoch": 0.8089887640449438,
+      "grad_norm": 7.092017650604248,
+      "learning_rate": 8.139534883720931e-06,
+      "logits/chosen": -0.6167936325073242,
+      "logits/rejected": -0.6189281940460205,
+      "logps/chosen": -1467.26904296875,
+      "logps/rejected": -1492.12890625,
+      "loss": 0.7104,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": -0.042918525636196136,
+      "rewards/margins": -0.0062050605192780495,
+      "rewards/rejected": -0.03671346232295036,
+      "step": 180
+    },
+    {
+      "epoch": 0.8539325842696629,
+      "grad_norm": 9.236512184143066,
+      "learning_rate": 7.9734219269103e-06,
+      "logits/chosen": -0.6129944920539856,
+      "logits/rejected": -0.6186620593070984,
+      "logps/chosen": -1476.5726318359375,
+      "logps/rejected": -1479.4190673828125,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.11343568563461304,
+      "rewards/margins": 0.0183081217110157,
+      "rewards/rejected": -0.13174381852149963,
+      "step": 190
+    },
+    {
+      "epoch": 0.898876404494382,
+      "grad_norm": 8.727457046508789,
+      "learning_rate": 7.807308970099668e-06,
+      "logits/chosen": -0.6297080516815186,
+      "logits/rejected": -0.6237484216690063,
+      "logps/chosen": -1487.1334228515625,
+      "logps/rejected": -1459.8831787109375,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.06383897364139557,
+      "rewards/margins": 0.05165405198931694,
+      "rewards/rejected": -0.1154930368065834,
+      "step": 200
+    },
+    {
+      "epoch": 0.9438202247191011,
+      "grad_norm": 8.576679229736328,
+      "learning_rate": 7.641196013289037e-06,
+      "logits/chosen": -0.6135516166687012,
+      "logits/rejected": -0.6387461423873901,
+      "logps/chosen": -1464.3004150390625,
+      "logps/rejected": -1465.748291015625,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.4906249940395355,
+      "rewards/chosen": 0.04854054003953934,
+      "rewards/margins": 0.009796160273253918,
+      "rewards/rejected": 0.038744375109672546,
+      "step": 210
+    },
+    {
+      "epoch": 0.9887640449438202,
+      "grad_norm": 7.046853542327881,
+      "learning_rate": 7.475083056478406e-06,
+      "logits/chosen": -0.6337479948997498,
+      "logits/rejected": -0.6330554485321045,
+      "logps/chosen": -1460.4818115234375,
+      "logps/rejected": -1454.669189453125,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08278223127126694,
+      "rewards/margins": 0.04179311543703079,
+      "rewards/rejected": -0.12457535415887833,
+      "step": 220
+    },
+    {
+      "epoch": 1.0314606741573034,
+      "grad_norm": 6.021562576293945,
+      "learning_rate": 7.308970099667775e-06,
+      "logits/chosen": -0.6283758282661438,
+      "logits/rejected": -0.6265605092048645,
+      "logps/chosen": -1488.2630615234375,
+      "logps/rejected": -1481.769775390625,
+      "loss": 0.5947,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.006120601668953896,
+      "rewards/margins": 0.2608007490634918,
+      "rewards/rejected": -0.25468015670776367,
+      "step": 230
+    },
+    {
+      "epoch": 1.0764044943820226,
+      "grad_norm": 8.284282684326172,
+      "learning_rate": 7.1428571428571436e-06,
+      "logits/chosen": -0.6591601371765137,
+      "logits/rejected": -0.652434229850769,
+      "logps/chosen": -1491.092529296875,
+      "logps/rejected": -1460.5125732421875,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1593593955039978,
+      "rewards/margins": 0.3331313133239746,
+      "rewards/rejected": -0.1737719029188156,
+      "step": 240
+    },
+    {
+      "epoch": 1.1213483146067416,
+      "grad_norm": 8.415833473205566,
+      "learning_rate": 6.976744186046513e-06,
+      "logits/chosen": -0.6509793996810913,
+      "logits/rejected": -0.64632248878479,
+      "logps/chosen": -1457.5777587890625,
+      "logps/rejected": -1468.1063232421875,
+      "loss": 0.5393,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.19115446507930756,
+      "rewards/margins": 0.37512803077697754,
+      "rewards/rejected": -0.1839735507965088,
+      "step": 250
+    },
+    {
+      "epoch": 1.1662921348314608,
+      "grad_norm": 6.646719932556152,
+      "learning_rate": 6.810631229235881e-06,
+      "logits/chosen": -0.6484296917915344,
+      "logits/rejected": -0.6388376951217651,
+      "logps/chosen": -1499.72998046875,
+      "logps/rejected": -1491.3359375,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.02073713205754757,
+      "rewards/margins": 0.46458253264427185,
+      "rewards/rejected": -0.48531967401504517,
+      "step": 260
+    },
+    {
+      "epoch": 1.2112359550561798,
+      "grad_norm": 7.409927845001221,
+      "learning_rate": 6.64451827242525e-06,
+      "logits/chosen": -0.6177853345870972,
+      "logits/rejected": -0.6329953670501709,
+      "logps/chosen": -1474.36474609375,
+      "logps/rejected": -1496.6304931640625,
+      "loss": 0.5096,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.14079628884792328,
+      "rewards/margins": 0.45578232407569885,
+      "rewards/rejected": -0.5965785980224609,
+      "step": 270
+    },
+    {
+      "epoch": 1.256179775280899,
+      "grad_norm": 5.9450225830078125,
+      "learning_rate": 6.4784053156146185e-06,
+      "logits/chosen": -0.6525323987007141,
+      "logits/rejected": -0.645168662071228,
+      "logps/chosen": -1508.730712890625,
+      "logps/rejected": -1497.562744140625,
+      "loss": 0.517,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.2821454405784607,
+      "rewards/margins": 0.4555455148220062,
+      "rewards/rejected": -0.7376910448074341,
+      "step": 280
+    },
+    {
+      "epoch": 1.301123595505618,
+      "grad_norm": 6.388819217681885,
+      "learning_rate": 6.312292358803988e-06,
+      "logits/chosen": -0.6514378190040588,
+      "logits/rejected": -0.6536304354667664,
+      "logps/chosen": -1464.6539306640625,
+      "logps/rejected": -1477.315185546875,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": -0.15214049816131592,
+      "rewards/margins": 0.48767295479774475,
+      "rewards/rejected": -0.6398134231567383,
+      "step": 290
+    },
+    {
+      "epoch": 1.346067415730337,
+      "grad_norm": 6.948678970336914,
+      "learning_rate": 6.146179401993356e-06,
+      "logits/chosen": -0.6365095973014832,
+      "logits/rejected": -0.6101225018501282,
+      "logps/chosen": -1467.804931640625,
+      "logps/rejected": -1457.0943603515625,
+      "loss": 0.5226,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": -0.07389238476753235,
+      "rewards/margins": 0.45187896490097046,
+      "rewards/rejected": -0.5257713794708252,
+      "step": 300
+    },
+    {
+      "epoch": 1.3910112359550562,
+      "grad_norm": 6.829395771026611,
+      "learning_rate": 5.980066445182725e-06,
+      "logits/chosen": -0.6450862884521484,
+      "logits/rejected": -0.6496433615684509,
+      "logps/chosen": -1431.3052978515625,
+      "logps/rejected": -1472.2398681640625,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.06596250832080841,
+      "rewards/margins": 0.4646366536617279,
+      "rewards/rejected": -0.3986741900444031,
+      "step": 310
+    },
+    {
+      "epoch": 1.4359550561797754,
+      "grad_norm": 7.039060115814209,
+      "learning_rate": 5.8139534883720935e-06,
+      "logits/chosen": -0.6316220164299011,
+      "logits/rejected": -0.6179653406143188,
+      "logps/chosen": -1458.3851318359375,
+      "logps/rejected": -1472.72509765625,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2739182710647583,
+      "rewards/margins": 0.4745880663394928,
+      "rewards/rejected": -0.7485063076019287,
+      "step": 320
+    },
+    {
+      "epoch": 1.4808988764044944,
+      "grad_norm": 6.328307628631592,
+      "learning_rate": 5.647840531561463e-06,
+      "logits/chosen": -0.6379339098930359,
+      "logits/rejected": -0.6447880268096924,
+      "logps/chosen": -1484.3759765625,
+      "logps/rejected": -1490.268310546875,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1919710487127304,
+      "rewards/margins": 0.5375956296920776,
+      "rewards/rejected": -0.7295666933059692,
+      "step": 330
+    },
+    {
+      "epoch": 1.5258426966292133,
+      "grad_norm": 6.797203540802002,
+      "learning_rate": 5.481727574750831e-06,
+      "logits/chosen": -0.6540791392326355,
+      "logits/rejected": -0.6439436674118042,
+      "logps/chosen": -1445.4132080078125,
+      "logps/rejected": -1481.77294921875,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.012084199115633965,
+      "rewards/margins": 0.51849764585495,
+      "rewards/rejected": -0.5305818319320679,
+      "step": 340
+    },
+    {
+      "epoch": 1.5707865168539326,
+      "grad_norm": 6.147586822509766,
+      "learning_rate": 5.3156146179402e-06,
+      "logits/chosen": -0.6325907707214355,
+      "logits/rejected": -0.6374093294143677,
+      "logps/chosen": -1445.081298828125,
+      "logps/rejected": -1476.953857421875,
+      "loss": 0.4924,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": -0.1005701795220375,
+      "rewards/margins": 0.5271730422973633,
+      "rewards/rejected": -0.6277432441711426,
+      "step": 350
+    },
+    {
+      "epoch": 1.6157303370786518,
+      "grad_norm": 7.819535255432129,
+      "learning_rate": 5.149501661129569e-06,
+      "logits/chosen": -0.6434189081192017,
+      "logits/rejected": -0.6285640597343445,
+      "logps/chosen": -1475.154296875,
+      "logps/rejected": -1454.4254150390625,
+      "loss": 0.4818,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.3274011015892029,
+      "rewards/margins": 0.5485560297966003,
+      "rewards/rejected": -0.8759571313858032,
+      "step": 360
+    },
+    {
+      "epoch": 1.6606741573033708,
+      "grad_norm": 6.248306751251221,
+      "learning_rate": 4.983388704318937e-06,
+      "logits/chosen": -0.64997398853302,
+      "logits/rejected": -0.645468533039093,
+      "logps/chosen": -1450.2716064453125,
+      "logps/rejected": -1451.9000244140625,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.3560061454772949,
+      "rewards/margins": 0.5591068863868713,
+      "rewards/rejected": -0.915113091468811,
+      "step": 370
+    },
+    {
+      "epoch": 1.7056179775280897,
+      "grad_norm": 7.458978176116943,
+      "learning_rate": 4.817275747508306e-06,
+      "logits/chosen": -0.6436145305633545,
+      "logits/rejected": -0.6464294195175171,
+      "logps/chosen": -1480.620849609375,
+      "logps/rejected": -1482.4827880859375,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3419143557548523,
+      "rewards/margins": 0.5034736394882202,
+      "rewards/rejected": -0.8453879356384277,
+      "step": 380
+    },
+    {
+      "epoch": 1.750561797752809,
+      "grad_norm": 6.304107666015625,
+      "learning_rate": 4.651162790697675e-06,
+      "logits/chosen": -0.6337984800338745,
+      "logits/rejected": -0.6419006586074829,
+      "logps/chosen": -1457.7503662109375,
+      "logps/rejected": -1450.752197265625,
+      "loss": 0.4727,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1469828486442566,
+      "rewards/margins": 0.5898675322532654,
+      "rewards/rejected": -0.7368504405021667,
+      "step": 390
+    },
+    {
+      "epoch": 1.7955056179775282,
+      "grad_norm": 7.871026039123535,
+      "learning_rate": 4.4850498338870435e-06,
+      "logits/chosen": -0.6506339311599731,
+      "logits/rejected": -0.6489716172218323,
+      "logps/chosen": -1471.711181640625,
+      "logps/rejected": -1500.9840087890625,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.3221575617790222,
+      "rewards/margins": 0.5669270753860474,
+      "rewards/rejected": -0.8890846371650696,
+      "step": 400
+    },
+    {
+      "epoch": 1.8404494382022472,
+      "grad_norm": 7.770719528198242,
+      "learning_rate": 4.318936877076413e-06,
+      "logits/chosen": -0.633885383605957,
+      "logits/rejected": -0.635451078414917,
+      "logps/chosen": -1452.253662109375,
+      "logps/rejected": -1468.77392578125,
+      "loss": 0.4969,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": -0.414605051279068,
+      "rewards/margins": 0.5224484801292419,
+      "rewards/rejected": -0.9370535612106323,
+      "step": 410
+    },
+    {
+      "epoch": 1.8853932584269661,
+      "grad_norm": 6.401175022125244,
+      "learning_rate": 4.152823920265781e-06,
+      "logits/chosen": -0.6535300016403198,
+      "logits/rejected": -0.6600362658500671,
+      "logps/chosen": -1446.4879150390625,
+      "logps/rejected": -1480.5345458984375,
+      "loss": 0.4815,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.5065728425979614,
+      "rewards/margins": 0.5753783583641052,
+      "rewards/rejected": -1.0819512605667114,
+      "step": 420
+    },
+    {
+      "epoch": 1.9303370786516854,
+      "grad_norm": 6.45891809463501,
+      "learning_rate": 3.98671096345515e-06,
+      "logits/chosen": -0.6408799886703491,
+      "logits/rejected": -0.6440805792808533,
+      "logps/chosen": -1451.79638671875,
+      "logps/rejected": -1458.14794921875,
+      "loss": 0.477,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": -0.42773446440696716,
+      "rewards/margins": 0.591695249080658,
+      "rewards/rejected": -1.0194295644760132,
+      "step": 430
+    },
+    {
+      "epoch": 1.9752808988764046,
+      "grad_norm": 6.308041095733643,
+      "learning_rate": 3.8205980066445185e-06,
+      "logits/chosen": -0.660961389541626,
+      "logits/rejected": -0.6538819670677185,
+      "logps/chosen": -1483.5074462890625,
+      "logps/rejected": -1495.02734375,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.2519773840904236,
+      "rewards/margins": 0.572425365447998,
+      "rewards/rejected": -0.8244028091430664,
+      "step": 440
+    },
+    {
+      "epoch": 2.0179775280898875,
+      "grad_norm": 6.544996738433838,
+      "learning_rate": 3.6544850498338876e-06,
+      "logits/chosen": -0.6604510545730591,
+      "logits/rejected": -0.6692507266998291,
+      "logps/chosen": -1480.155517578125,
+      "logps/rejected": -1485.3016357421875,
+      "loss": 0.425,
+      "rewards/accuracies": 0.9013158082962036,
+      "rewards/chosen": -0.029593899846076965,
+      "rewards/margins": 0.7573708891868591,
+      "rewards/rejected": -0.7869648337364197,
+      "step": 450
+    },
+    {
+      "epoch": 2.0629213483146067,
+      "grad_norm": 6.268301486968994,
+      "learning_rate": 3.4883720930232564e-06,
+      "logits/chosen": -0.6653432846069336,
+      "logits/rejected": -0.6545718908309937,
+      "logps/chosen": -1456.055419921875,
+      "logps/rejected": -1484.5994873046875,
+      "loss": 0.3421,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 0.03443983197212219,
+      "rewards/margins": 0.9891597628593445,
+      "rewards/rejected": -0.9547199010848999,
+      "step": 460
+    },
+    {
+      "epoch": 2.107865168539326,
+      "grad_norm": 6.637356281280518,
+      "learning_rate": 3.322259136212625e-06,
+      "logits/chosen": -0.669964611530304,
+      "logits/rejected": -0.6756006479263306,
+      "logps/chosen": -1436.8699951171875,
+      "logps/rejected": -1450.5247802734375,
+      "loss": 0.3371,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.20131292939186096,
+      "rewards/margins": 1.0167487859725952,
+      "rewards/rejected": -1.2180618047714233,
+      "step": 470
+    },
+    {
+      "epoch": 2.152808988764045,
+      "grad_norm": 5.162894248962402,
+      "learning_rate": 3.156146179401994e-06,
+      "logits/chosen": -0.6826899647712708,
+      "logits/rejected": -0.6783708333969116,
+      "logps/chosen": -1470.298583984375,
+      "logps/rejected": -1493.745849609375,
+      "loss": 0.3365,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.25590771436691284,
+      "rewards/margins": 1.0207464694976807,
+      "rewards/rejected": -1.2766541242599487,
+      "step": 480
+    },
+    {
+      "epoch": 2.197752808988764,
+      "grad_norm": 5.52967643737793,
+      "learning_rate": 2.9900332225913626e-06,
+      "logits/chosen": -0.6881505250930786,
+      "logits/rejected": -0.682505190372467,
+      "logps/chosen": -1448.319580078125,
+      "logps/rejected": -1477.046630859375,
+      "loss": 0.3315,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.37032753229141235,
+      "rewards/margins": 1.0482890605926514,
+      "rewards/rejected": -1.4186166524887085,
+      "step": 490
+    },
+    {
+      "epoch": 2.242696629213483,
+      "grad_norm": 5.7678399085998535,
+      "learning_rate": 2.8239202657807313e-06,
+      "logits/chosen": -0.7020605206489563,
+      "logits/rejected": -0.6847004294395447,
+      "logps/chosen": -1463.9556884765625,
+      "logps/rejected": -1515.1861572265625,
+      "loss": 0.3295,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": -0.5403343439102173,
+      "rewards/margins": 1.0608083009719849,
+      "rewards/rejected": -1.6011426448822021,
+      "step": 500
+    },
+    {
+      "epoch": 2.2876404494382023,
+      "grad_norm": 6.095832347869873,
+      "learning_rate": 2.6578073089701e-06,
+      "logits/chosen": -0.6875178217887878,
+      "logits/rejected": -0.6721357107162476,
+      "logps/chosen": -1503.990234375,
+      "logps/rejected": -1523.0850830078125,
+      "loss": 0.3239,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.7074209451675415,
+      "rewards/margins": 1.0821547508239746,
+      "rewards/rejected": -1.7895758152008057,
+      "step": 510
+    },
+    {
+      "epoch": 2.3325842696629215,
+      "grad_norm": 5.140925884246826,
+      "learning_rate": 2.4916943521594684e-06,
+      "logits/chosen": -0.6903117299079895,
+      "logits/rejected": -0.6828970909118652,
+      "logps/chosen": -1494.531494140625,
+      "logps/rejected": -1525.615966796875,
+      "loss": 0.3329,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.7834824323654175,
+      "rewards/margins": 1.0618258714675903,
+      "rewards/rejected": -1.845308542251587,
+      "step": 520
+    },
+    {
+      "epoch": 2.3775280898876403,
+      "grad_norm": 5.589333534240723,
+      "learning_rate": 2.3255813953488376e-06,
+      "logits/chosen": -0.6674150824546814,
+      "logits/rejected": -0.6766713857650757,
+      "logps/chosen": -1438.752197265625,
+      "logps/rejected": -1477.069091796875,
+      "loss": 0.3327,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.8180882334709167,
+      "rewards/margins": 1.0446465015411377,
+      "rewards/rejected": -1.8627347946166992,
+      "step": 530
+    },
+    {
+      "epoch": 2.4224719101123595,
+      "grad_norm": 5.4515814781188965,
+      "learning_rate": 2.1594684385382063e-06,
+      "logits/chosen": -0.6770123839378357,
+      "logits/rejected": -0.6860142350196838,
+      "logps/chosen": -1475.977294921875,
+      "logps/rejected": -1495.72216796875,
+      "loss": 0.3264,
+      "rewards/accuracies": 0.965624988079071,
+      "rewards/chosen": -0.7763740420341492,
+      "rewards/margins": 1.0760669708251953,
+      "rewards/rejected": -1.8524410724639893,
+      "step": 540
+    },
+    {
+      "epoch": 2.4674157303370787,
+      "grad_norm": 6.506235122680664,
+      "learning_rate": 1.993355481727575e-06,
+      "logits/chosen": -0.6790199279785156,
+      "logits/rejected": -0.677984356880188,
+      "logps/chosen": -1483.64013671875,
+      "logps/rejected": -1499.1666259765625,
+      "loss": 0.3352,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": -0.6030625104904175,
+      "rewards/margins": 1.0407390594482422,
+      "rewards/rejected": -1.6438014507293701,
+      "step": 550
+    },
+    {
+      "epoch": 2.512359550561798,
+      "grad_norm": 4.6371331214904785,
+      "learning_rate": 1.8272425249169438e-06,
+      "logits/chosen": -0.6902989149093628,
+      "logits/rejected": -0.6996210217475891,
+      "logps/chosen": -1499.473388671875,
+      "logps/rejected": -1493.6162109375,
+      "loss": 0.3142,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": -0.38727277517318726,
+      "rewards/margins": 1.1097298860549927,
+      "rewards/rejected": -1.4970027208328247,
+      "step": 560
+    },
+    {
+      "epoch": 2.5573033707865167,
+      "grad_norm": 5.494938850402832,
+      "learning_rate": 1.6611295681063126e-06,
+      "logits/chosen": -0.6864480972290039,
+      "logits/rejected": -0.6946516036987305,
+      "logps/chosen": -1488.853515625,
+      "logps/rejected": -1484.619384765625,
+      "loss": 0.3085,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.39798271656036377,
+      "rewards/margins": 1.1385451555252075,
+      "rewards/rejected": -1.5365278720855713,
+      "step": 570
+    },
+    {
+      "epoch": 2.602247191011236,
+      "grad_norm": 5.5001959800720215,
+      "learning_rate": 1.4950166112956813e-06,
+      "logits/chosen": -0.689426600933075,
+      "logits/rejected": -0.6980553865432739,
+      "logps/chosen": -1462.147705078125,
+      "logps/rejected": -1470.9658203125,
+      "loss": 0.309,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.42978817224502563,
+      "rewards/margins": 1.1281216144561768,
+      "rewards/rejected": -1.5579097270965576,
+      "step": 580
+    },
+    {
+      "epoch": 2.647191011235955,
+      "grad_norm": 5.16894006729126,
+      "learning_rate": 1.32890365448505e-06,
+      "logits/chosen": -0.6979320645332336,
+      "logits/rejected": -0.6933678984642029,
+      "logps/chosen": -1492.7177734375,
+      "logps/rejected": -1465.2548828125,
+      "loss": 0.3171,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.3818231225013733,
+      "rewards/margins": 1.1230003833770752,
+      "rewards/rejected": -1.5048235654830933,
+      "step": 590
+    },
+    {
+      "epoch": 2.692134831460674,
+      "grad_norm": 5.261245250701904,
+      "learning_rate": 1.1627906976744188e-06,
+      "logits/chosen": -0.6973509788513184,
+      "logits/rejected": -0.6787486672401428,
+      "logps/chosen": -1474.0703125,
+      "logps/rejected": -1494.2147216796875,
+      "loss": 0.3119,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.23206683993339539,
+      "rewards/margins": 1.1345573663711548,
+      "rewards/rejected": -1.366624116897583,
+      "step": 600
+    },
+    {
+      "epoch": 2.737078651685393,
+      "grad_norm": 4.852719783782959,
+      "learning_rate": 9.966777408637875e-07,
+      "logits/chosen": -0.7001398205757141,
+      "logits/rejected": -0.6987439393997192,
+      "logps/chosen": -1424.858154296875,
+      "logps/rejected": -1409.1744384765625,
+      "loss": 0.3177,
+      "rewards/accuracies": 0.9593750238418579,
+      "rewards/chosen": -0.19675633311271667,
+      "rewards/margins": 1.1072434186935425,
+      "rewards/rejected": -1.303999662399292,
+      "step": 610
+    },
+    {
+      "epoch": 2.7820224719101123,
+      "grad_norm": 6.0668745040893555,
+      "learning_rate": 8.305647840531563e-07,
+      "logits/chosen": -0.7101883888244629,
+      "logits/rejected": -0.7109605073928833,
+      "logps/chosen": -1480.221435546875,
+      "logps/rejected": -1482.60546875,
+      "loss": 0.3195,
+      "rewards/accuracies": 0.965624988079071,
+      "rewards/chosen": -0.19247181713581085,
+      "rewards/margins": 1.0987659692764282,
+      "rewards/rejected": -1.2912375926971436,
+      "step": 620
+    },
+    {
+      "epoch": 2.8269662921348315,
+      "grad_norm": 5.055160999298096,
+      "learning_rate": 6.64451827242525e-07,
+      "logits/chosen": -0.7024872899055481,
+      "logits/rejected": -0.6898430585861206,
+      "logps/chosen": -1450.166015625,
+      "logps/rejected": -1479.9564208984375,
+      "loss": 0.3166,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": -0.2648771405220032,
+      "rewards/margins": 1.1128193140029907,
+      "rewards/rejected": -1.3776965141296387,
+      "step": 630
+    },
+    {
+      "epoch": 2.8719101123595507,
+      "grad_norm": 5.179624557495117,
+      "learning_rate": 4.983388704318938e-07,
+      "logits/chosen": -0.7008506059646606,
+      "logits/rejected": -0.6875109672546387,
+      "logps/chosen": -1489.664306640625,
+      "logps/rejected": -1495.178466796875,
+      "loss": 0.3158,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.3089246153831482,
+      "rewards/margins": 1.1327073574066162,
+      "rewards/rejected": -1.4416319131851196,
+      "step": 640
+    },
+    {
+      "epoch": 2.9168539325842695,
+      "grad_norm": 6.70164680480957,
+      "learning_rate": 3.322259136212625e-07,
+      "logits/chosen": -0.6998602151870728,
+      "logits/rejected": -0.7018693685531616,
+      "logps/chosen": -1458.418212890625,
+      "logps/rejected": -1455.3675537109375,
+      "loss": 0.3231,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.26613983511924744,
+      "rewards/margins": 1.1134045124053955,
+      "rewards/rejected": -1.3795442581176758,
+      "step": 650
+    },
+    {
+      "epoch": 2.9617977528089887,
+      "grad_norm": 5.529868125915527,
+      "learning_rate": 1.6611295681063126e-07,
+      "logits/chosen": -0.7056189775466919,
+      "logits/rejected": -0.7010918259620667,
+      "logps/chosen": -1448.5848388671875,
+      "logps/rejected": -1449.268310546875,
+      "loss": 0.3193,
+      "rewards/accuracies": 0.965624988079071,
+      "rewards/chosen": -0.2537583112716675,
+      "rewards/margins": 1.1171658039093018,
+      "rewards/rejected": -1.3709241151809692,
+      "step": 660
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 669,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

Preference_Set_Qwen2.5-3B-Instruct_INFV_ref_as_gt_True_IterRet_individual_recall_True_top_k_30/Qwen/Qwen2.5-3B-Instruct/method_agent/checkpoint-669/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d65df17b30d9c0fee0b16beb563e4fd4ae603d6a11683ef06705512d4b6bf93
+size 7057