Update context length

configuration_llama.py

@@ -114,7 +114,7 @@ class LlamaConfig(PretrainedConfig):
         num_attention_heads=40,
         num_key_value_heads=40,
         hidden_act="silu",
-        max_position_embeddings=10240,
+        max_position_embeddings=32768,
         initializer_range=0.02,
         rms_norm_eps=1e-05,
         use_cache=True,
@@ -124,7 +124,7 @@ class LlamaConfig(PretrainedConfig):
         pretraining_tp=1,
         tie_word_embeddings=False,
         rope_theta=10000,
-        rope_scaling="yarn",
+        rope_scaling=None,
         attention_bias=False,
         **kwargs,
     ):