Training checkpoint at step 1000

config.json

@@ -7,11 +7,11 @@
   ],
   "attend_to_previous_input": false,
   "base_model_config": null,
-  "base_model_name": "HuggingFaceTB/SmolLM2-360M",
+  "base_model_name": "meta-llama/Llama-3.2-1B",
   "constant_depth": false,
   "correction": true,
   "d_mem": 64,
-  "dtype": "float32",
+  "dtype": "bfloat16",
   "freeze_mem": false,
   "gating": false,
   "layers_attr": "model.layers",
@@ -21,12 +21,12 @@
   "noisy_halting": false,
   "num_mem_tokens": 32,
   "segment_alignment": "left",
-  "segment_size": 256,
-  "sliding_window": false,
+  "segment_size": 1024,
+  "sliding_window": true,
   "time_penalty": 0.0,
-  "transformers_version": "4.57.1",
+  "transformers_version": "4.57.3",
   "use_denom": true,
-  "use_sink": false,
+  "use_sink": true,
   "wrap_layers": null,
   "wrap_pos": false,
   "auto_map": {