Enable MiniMax models and wire their image/video request format

evaluate.py

@@ -20,6 +20,20 @@
 with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "src", "utils", "allowed_models.json")) as f:
     ALLOWED_MODELS = json.load(f)["allowed_models"]
 
+MINIMAX_VIDEO_MODEL = "minimax/MiniMax-M3"
+VIDEO_MODELS = [
+    "gemini/gemini-1.5-pro-002",
+    "gemini/gemini-2.0-flash-exp",
+    "gemini/gemini-2.0-pro-exp-02-05",
+    MINIMAX_VIDEO_MODEL,
+]
+
+
+def _create_video_model(model_name: str):
+    if model_name == MINIMAX_VIDEO_MODEL:
+        return LiteLLMWrapper(model_name=model_name, temperature=0.0)
+    return GeminiWrapper(model_name=model_name, temperature=0.0)
+
 
 def combine_results(output_folder: str, combined_file: str, results: Dict[str, Dict]) -> None:
     """
@@ -359,9 +373,7 @@ def main():
                        default='azure/gpt-4o',
                        help='Select the AI model to use for text evaluation')
     parser.add_argument('--model_video', type=str,
-                       choices=['gemini/gemini-1.5-pro-002',
-                                'gemini/gemini-2.0-flash-exp',
-                                'gemini/gemini-2.0-pro-exp-02-05'],
+                       choices=VIDEO_MODELS,
                        default='gemini/gemini-1.5-pro-002',
                        help='Select the AI model to use for video evaluation')
     parser.add_argument('--model_image', type=str,
@@ -385,10 +397,7 @@ def main():
         model_name=args.model_text,
         temperature=0.0,
     )
-    video_model = GeminiWrapper(
-        model_name=args.model_video,
-        temperature=0.0,
-    )
+    video_model = _create_video_model(args.model_video)
     image_model = LiteLLMWrapper(
         model_name=args.model_image,
         temperature=0.0,

mllm_tools/litellm.py

@@ -12,6 +12,9 @@
 
 load_dotenv()
 
+MINIMAX_MULTIMODAL_MODELS = {"minimax/minimax-m3"}
+
+
 class LiteLLMWrapper:
     """Wrapper for LiteLLM to support multiple models and logging"""
 
@@ -148,8 +151,39 @@ def __call__(self, messages: List[Dict[str, Any]], metadata: Optional[Dict[str,
                         })
                     else:
                         raise ValueError("For GPT, only text and image inferencing are supported")
+                elif self.model_name.lower().startswith("minimax/"):
+                    if self.model_name.lower() not in MINIMAX_MULTIMODAL_MODELS:
+                        raise ValueError(f"{self.model_name} only supports text input")
+
+                    # MiniMax-M3 uses OpenAI-compatible image and video blocks.
+                    if msg["type"] == "image":
+                        formatted_messages.append({
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": data_url
+                                    }
+                                }
+                            ]
+                        })
+                    elif msg["type"] == "video":
+                        formatted_messages.append({
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "video_url",
+                                    "video_url": {
+                                        "url": data_url
+                                    }
+                                }
+                            ]
+                        })
+                    else:
+                        raise ValueError("For MiniMax, only text, image and video inferencing are supported")
                 else:
-                    raise ValueError("Only support Gemini and Gpt for Multimodal capability now")
+                    raise ValueError("Only support Gemini, Gpt and MiniMax for Multimodal capability now")
 
         try:
             # if it's openai o series model, set temperature to None and reasoning_effort to "medium"
@@ -190,4 +224,4 @@ def __call__(self, messages: List[Dict[str, Any]], metadata: Optional[Dict[str,
             return str(e)
 
 if __name__ == "__main__":
-    pass
+    pass

src/utils/allowed_models.json

@@ -13,6 +13,8 @@
     "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
     "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
     "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0",
-    "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0"
+    "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+    "minimax/MiniMax-M3",
+    "minimax/MiniMax-M2.7"
   ]
 }

tests/test_minimax_support.py

@@ -0,0 +1,120 @@
+import sys
+import types
+import unittest
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+
+fake_litellm = types.ModuleType("litellm")
+fake_litellm.completion = Mock(
+    return_value=SimpleNamespace(
+        choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))]
+    )
+)
+fake_litellm.completion_cost = Mock(return_value=0)
+fake_litellm.success_callback = []
+fake_litellm.failure_callback = []
+sys.modules["litellm"] = fake_litellm
+
+fake_dotenv = types.ModuleType("dotenv")
+fake_dotenv.load_dotenv = Mock()
+sys.modules["dotenv"] = fake_dotenv
+
+from mllm_tools.litellm import LiteLLMWrapper
+
+
+class FakeGeminiWrapper:
+    def __init__(self, model_name, temperature):
+        self.model_name = model_name
+        self.temperature = temperature
+
+
+fake_moviepy = types.ModuleType("moviepy")
+fake_moviepy.VideoFileClip = object
+sys.modules["moviepy"] = fake_moviepy
+
+fake_gemini = types.ModuleType("mllm_tools.gemini")
+fake_gemini.GeminiWrapper = FakeGeminiWrapper
+sys.modules["mllm_tools.gemini"] = fake_gemini
+
+fake_eval_utils = types.ModuleType("eval_suite.utils")
+fake_eval_utils.calculate_geometric_mean = Mock()
+sys.modules["eval_suite.utils"] = fake_eval_utils
+
+fake_text_utils = types.ModuleType("eval_suite.text_utils")
+fake_text_utils.parse_srt_to_text = Mock()
+fake_text_utils.fix_transcript = Mock()
+fake_text_utils.evaluate_text = Mock()
+sys.modules["eval_suite.text_utils"] = fake_text_utils
+
+fake_video_utils = types.ModuleType("eval_suite.video_utils")
+fake_video_utils.evaluate_video_chunk_new = Mock()
+sys.modules["eval_suite.video_utils"] = fake_video_utils
+
+fake_image_utils = types.ModuleType("eval_suite.image_utils")
+fake_image_utils.evaluate_sampled_images = Mock()
+sys.modules["eval_suite.image_utils"] = fake_image_utils
+
+import evaluate
+
+
+class MiniMaxSupportTest(unittest.TestCase):
+    def setUp(self):
+        fake_litellm.completion.reset_mock()
+
+    def test_minimax_m3_formats_image_and_video_inputs(self):
+        model = LiteLLMWrapper(
+            model_name="minimax/MiniMax-M3",
+            use_langfuse=False,
+        )
+
+        result = model([
+            {"type": "image", "content": "https://example.com/image.png"},
+            {"type": "video", "content": "https://example.com/video.mp4"},
+        ])
+
+        self.assertEqual(result, "ok")
+        messages = fake_litellm.completion.call_args.kwargs["messages"]
+        self.assertEqual(
+            messages[0]["content"][0],
+            {
+                "type": "image_url",
+                "image_url": {"url": "https://example.com/image.png"},
+            },
+        )
+        self.assertEqual(
+            messages[1]["content"][0],
+            {
+                "type": "video_url",
+                "video_url": {"url": "https://example.com/video.mp4"},
+            },
+        )
+
+    def test_minimax_m27_rejects_media_inputs(self):
+        model = LiteLLMWrapper(
+            model_name="minimax/MiniMax-M2.7",
+            use_langfuse=False,
+        )
+
+        with self.assertRaisesRegex(ValueError, "only supports text input"):
+            model([{"type": "image", "content": "https://example.com/image.png"}])
+
+        fake_litellm.completion.assert_not_called()
+
+    def test_minimax_m3_is_available_for_video_evaluation(self):
+        self.assertIn("minimax/MiniMax-M3", evaluate.VIDEO_MODELS)
+        self.assertNotIn("minimax/MiniMax-M2.7", evaluate.VIDEO_MODELS)
+        model = evaluate._create_video_model("minimax/MiniMax-M3")
+
+        self.assertIsInstance(model, LiteLLMWrapper)
+        self.assertEqual(model.model_name, "minimax/MiniMax-M3")
+
+    def test_existing_video_models_keep_their_wrapper(self):
+        model = evaluate._create_video_model("gemini/gemini-1.5-pro-002")
+
+        self.assertIsInstance(model, FakeGeminiWrapper)
+        self.assertEqual(model.model_name, "gemini/gemini-1.5-pro-002")
+
+
+if __name__ == "__main__":
+    unittest.main()