Patched it to work again. https://github.com/Q-Future/Q-Align/issues/31#issuecomment-2561704943

README.md

@@ -3,6 +3,13 @@ license: mit
 pipeline_tag: zero-shot-image-classification
 ---
 
+## This fork
+This fork fixes some issues with the newest versions of transformers.
+[https://github.com/Q-Future/Q-Align/issues/31](https://github.com/Q-Future/Q-Align/issues/31#issuecomment-2561704943)
+
+
+## Upstream
+
 The model that corresponds to Q-Align (ICML2024).
 
 ## Quick Start with AutoModel

config.json

@@ -33,6 +33,7 @@
   "transformers_version": "4.31.0",
   "tune_visual_abstractor": true,
   "use_cache": true,
+  "mlp_bias": false,
   "visual_abstractor_lr": null,
   "visual_config": {
     "visual_abstractor": {

modeling_llama2.py

@@ -22,8 +22,12 @@ from transformers.models.llama.modeling_llama import *
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
-from .modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa
-from .configuration_mplug_owl2 import LlamaConfig
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa
+from transformers.models.llama.configuration_llama import LlamaConfig
+
+#from .configuration_mplug_owl2 import LlamaConfig
+
 
 class MultiwayNetwork(nn.Module):
 
@@ -31,14 +35,14 @@ class MultiwayNetwork(nn.Module):
         super(MultiwayNetwork, self).__init__()
 
         self.multiway = torch.nn.ModuleList([module_provider() for _ in range(num_multiway)])
-    
+
     def forward(self, hidden_states, multiway_indices):
 
         if len(self.multiway) == 1:
             return self.multiway[0](hidden_states)
 
         output_hidden_states = torch.empty_like(hidden_states)
-        
+
         for idx, subway in enumerate(self.multiway):
             local_indices = multiway_indices.eq(idx).nonzero(as_tuple=True)
             hidden = hidden_states[local_indices].unsqueeze(1).contiguous()
@@ -48,9 +52,9 @@ class MultiwayNetwork(nn.Module):
                     output = output[0]
                 output = output.squeeze(1)
                 output_hidden_states[local_indices] = output
-        
+
         return output_hidden_states.contiguous()
-    
+
 
 class LlamaAttention(nn.Module):
     """Multi-headed attention from 'Attention Is All You Need' paper"""
@@ -65,7 +69,7 @@ class LlamaAttention(nn.Module):
                 "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
                 "when creating this class."
             )
-            
+
         self.attention_dropout = config.attention_dropout
         self.hidden_size = config.hidden_size
         self.num_heads = config.num_attention_heads
@@ -145,7 +149,8 @@ class LlamaAttention(nn.Module):
         kv_seq_len = key_states.shape[-2]
         if past_key_value is not None:
             kv_seq_len += past_key_value[0].shape[-2]
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        #cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        cos, sin = self.rotary_emb(value_states, position_ids)
         query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
 
         if past_key_value is not None:
@@ -193,7 +198,7 @@ class LlamaAttention(nn.Module):
             attn_weights = None
 
         return attn_output, attn_weights, past_key_value
-    
+
 
 class LlamaFlashAttention2(LlamaAttention):
     """
@@ -248,7 +253,8 @@ class LlamaFlashAttention2(LlamaAttention):
         kv_seq_len = key_states.shape[-2]
         if past_key_value is not None:
             kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        #cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        cos, sin = self.rotary_emb(value_states, position_ids)
         query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
 
         if past_key_value is not None:
@@ -446,7 +452,8 @@ class LlamaSdpaAttention(LlamaAttention):
         kv_seq_len = key_states.shape[-2]
         if past_key_value is not None:
             kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        #cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        cos, sin = self.rotary_emb(value_states, position_ids)
 
         query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
 
@@ -596,7 +603,7 @@ def model_forward(
         batch_size, seq_length, _ = inputs_embeds.shape
     else:
         raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")
-    
+
     seq_length_with_past = seq_length
     past_key_values_length = 0
 
@@ -620,11 +627,11 @@ def model_forward(
         attention_mask = torch.ones(
             (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device
         )
-        
-    if self._use_flash_attention_2:
+
+    if False: #self._use_flash_attention_2:
             # 2d mask is passed through the layers
             attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
-    elif self._use_sdpa and not output_attentions:
+    elif False: #self._use_sdpa and not output_attentions:
             # output_attentions=True can not be supported when using SDPA, and we fall back on
             # the manual implementation that requires a 4D causal mask in all cases.
             attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
@@ -814,7 +821,7 @@ def replace_llama_modality_adaptive():
     transformers.models.llama.modeling_llama.LlamaModel.forward = model_forward
     transformers.models.llama.modeling_llama.LlamaForCausalLM.forward = causal_model_forward
 
-    
+
 if __name__ == "__main__":
     replace_llama_modality_adaptive()
     config = transformers.LlamaConfig.from_pretrained('/cpfs01/shared/public/test/vicuna-7b-v1.5/')