Upload projectors.py with huggingface_hub

projectors.py

@@ -0,0 +1,93 @@
+"""
+Projection layers for multimodal fusion
+"""
+import torch
+import torch.nn as nn
+from typing import Optional
+
+
+class VisionProjector(nn.Module):
+    """Projects vision features to language model embedding space"""
+    
+    def __init__(
+        self, 
+        vision_dim: int, 
+        language_dim: int, 
+        hidden_dim: Optional[int] = None,
+        dropout: float = 0.1
+    ):
+        super().__init__()
+        hidden_dim = hidden_dim or language_dim
+        
+        self.projector = nn.Sequential(
+            nn.Linear(vision_dim, hidden_dim),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim, language_dim),
+            nn.LayerNorm(language_dim)
+        )
+        
+        # Initialize weights
+        self._init_weights()
+        
+    def _init_weights(self):
+        """Initialize projection weights"""
+        for module in self.modules():
+            if isinstance(module, nn.Linear):
+                nn.init.normal_(module.weight, std=0.02)
+                if module.bias is not None:
+                    nn.init.zeros_(module.bias)
+            elif isinstance(module, nn.LayerNorm):
+                nn.init.ones_(module.weight)
+                nn.init.zeros_(module.bias)
+        
+    def forward(self, vision_features: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            vision_features: [batch_size, vision_dim]
+        Returns:
+            projected_features: [batch_size, language_dim]
+        """
+        return self.projector(vision_features)
+
+
+class AudioProjector(nn.Module):
+    """Projects audio features to language model embedding space"""
+    
+    def __init__(
+        self, 
+        audio_dim: int, 
+        language_dim: int,
+        dropout: float = 0.1
+    ):
+        super().__init__()
+        
+        self.projector = nn.Sequential(
+            nn.Linear(audio_dim, language_dim),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.LayerNorm(language_dim)
+        )
+        
+        # Initialize weights
+        self._init_weights()
+        
+    def _init_weights(self):
+        """Initialize projection weights"""
+        for module in self.modules():
+            if isinstance(module, nn.Linear):
+                nn.init.normal_(module.weight, std=0.02)
+                if module.bias is not None:
+                    nn.init.zeros_(module.bias)
+            elif isinstance(module, nn.LayerNorm):
+                nn.init.ones_(module.weight)
+                nn.init.zeros_(module.bias)
+        
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            audio_features: [batch_size, audio_dim]
+        Returns:
+            projected_features: [batch_size, language_dim]
+        """
+        return self.projector(audio_features)