ddh0
/

OrcaMaid-v2-FIX-13b-32k

Text Generation

text-generation-inference

Model card Files Files and versions

ddh0 commited on Dec 20, 2023

Commit

a281267

·

1 Parent(s): b8352de

Update context length

Files changed (1) hide show

config.json +3 -3

config.json CHANGED Viewed

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "ddh0/OrcaMaid-v2-FIX-13b-10k",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -15,7 +15,7 @@
   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 13824,
-  "max_position_embeddings": 10240,
   "model_type": "llama",
   "num_attention_heads": 40,
   "num_hidden_layers": 40,
@@ -24,7 +24,7 @@
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
-    "factor": 2.5,
     "original_max_position_embeddings": 4096,
     "type": "yarn",
     "finetuned": false

 {
+  "_name_or_path": "ddh0/OrcaMaid-v2-FIX-13b-32k",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 13824,
+  "max_position_embeddings": 32768,
   "model_type": "llama",
   "num_attention_heads": 40,
   "num_hidden_layers": 40,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
+    "factor": 8.0,
     "original_max_position_embeddings": 4096,
     "type": "yarn",
     "finetuned": false