diff --git a/evaluate.py b/evaluate.py index e6da363bc..b151848c0 100644 --- a/evaluate.py +++ b/evaluate.py @@ -20,6 +20,20 @@ with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "src", "utils", "allowed_models.json")) as f: ALLOWED_MODELS = json.load(f)["allowed_models"] +MINIMAX_VIDEO_MODEL = "minimax/MiniMax-M3" +VIDEO_MODELS = [ + "gemini/gemini-1.5-pro-002", + "gemini/gemini-2.0-flash-exp", + "gemini/gemini-2.0-pro-exp-02-05", + MINIMAX_VIDEO_MODEL, +] + + +def _create_video_model(model_name: str): + if model_name == MINIMAX_VIDEO_MODEL: + return LiteLLMWrapper(model_name=model_name, temperature=0.0) + return GeminiWrapper(model_name=model_name, temperature=0.0) + def combine_results(output_folder: str, combined_file: str, results: Dict[str, Dict]) -> None: """ @@ -359,9 +373,7 @@ def main(): default='azure/gpt-4o', help='Select the AI model to use for text evaluation') parser.add_argument('--model_video', type=str, - choices=['gemini/gemini-1.5-pro-002', - 'gemini/gemini-2.0-flash-exp', - 'gemini/gemini-2.0-pro-exp-02-05'], + choices=VIDEO_MODELS, default='gemini/gemini-1.5-pro-002', help='Select the AI model to use for video evaluation') parser.add_argument('--model_image', type=str, @@ -385,10 +397,7 @@ def main(): model_name=args.model_text, temperature=0.0, ) - video_model = GeminiWrapper( - model_name=args.model_video, - temperature=0.0, - ) + video_model = _create_video_model(args.model_video) image_model = LiteLLMWrapper( model_name=args.model_image, temperature=0.0, diff --git a/mllm_tools/litellm.py b/mllm_tools/litellm.py index 4873494ad..936c80857 100644 --- a/mllm_tools/litellm.py +++ b/mllm_tools/litellm.py @@ -12,6 +12,9 @@ load_dotenv() +MINIMAX_MULTIMODAL_MODELS = {"minimax/minimax-m3"} + + class LiteLLMWrapper: """Wrapper for LiteLLM to support multiple models and logging""" @@ -148,8 +151,39 @@ def __call__(self, messages: List[Dict[str, Any]], metadata: Optional[Dict[str, }) else: raise ValueError("For GPT, only text and image inferencing are supported") + elif self.model_name.lower().startswith("minimax/"): + if self.model_name.lower() not in MINIMAX_MULTIMODAL_MODELS: + raise ValueError(f"{self.model_name} only supports text input") + + # MiniMax-M3 uses OpenAI-compatible image and video blocks. + if msg["type"] == "image": + formatted_messages.append({ + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": data_url + } + } + ] + }) + elif msg["type"] == "video": + formatted_messages.append({ + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": { + "url": data_url + } + } + ] + }) + else: + raise ValueError("For MiniMax, only text, image and video inferencing are supported") else: - raise ValueError("Only support Gemini and Gpt for Multimodal capability now") + raise ValueError("Only support Gemini, Gpt and MiniMax for Multimodal capability now") try: # if it's openai o series model, set temperature to None and reasoning_effort to "medium" @@ -190,4 +224,4 @@ def __call__(self, messages: List[Dict[str, Any]], metadata: Optional[Dict[str, return str(e) if __name__ == "__main__": - pass \ No newline at end of file + pass diff --git a/src/utils/allowed_models.json b/src/utils/allowed_models.json index 2ca44ee76..35071367a 100644 --- a/src/utils/allowed_models.json +++ b/src/utils/allowed_models.json @@ -13,6 +13,8 @@ "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0", - "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0" + "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "minimax/MiniMax-M3", + "minimax/MiniMax-M2.7" ] } \ No newline at end of file diff --git a/tests/test_minimax_support.py b/tests/test_minimax_support.py new file mode 100644 index 000000000..f7e528409 --- /dev/null +++ b/tests/test_minimax_support.py @@ -0,0 +1,120 @@ +import sys +import types +import unittest +from types import SimpleNamespace +from unittest.mock import Mock + + +fake_litellm = types.ModuleType("litellm") +fake_litellm.completion = Mock( + return_value=SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ) +) +fake_litellm.completion_cost = Mock(return_value=0) +fake_litellm.success_callback = [] +fake_litellm.failure_callback = [] +sys.modules["litellm"] = fake_litellm + +fake_dotenv = types.ModuleType("dotenv") +fake_dotenv.load_dotenv = Mock() +sys.modules["dotenv"] = fake_dotenv + +from mllm_tools.litellm import LiteLLMWrapper + + +class FakeGeminiWrapper: + def __init__(self, model_name, temperature): + self.model_name = model_name + self.temperature = temperature + + +fake_moviepy = types.ModuleType("moviepy") +fake_moviepy.VideoFileClip = object +sys.modules["moviepy"] = fake_moviepy + +fake_gemini = types.ModuleType("mllm_tools.gemini") +fake_gemini.GeminiWrapper = FakeGeminiWrapper +sys.modules["mllm_tools.gemini"] = fake_gemini + +fake_eval_utils = types.ModuleType("eval_suite.utils") +fake_eval_utils.calculate_geometric_mean = Mock() +sys.modules["eval_suite.utils"] = fake_eval_utils + +fake_text_utils = types.ModuleType("eval_suite.text_utils") +fake_text_utils.parse_srt_to_text = Mock() +fake_text_utils.fix_transcript = Mock() +fake_text_utils.evaluate_text = Mock() +sys.modules["eval_suite.text_utils"] = fake_text_utils + +fake_video_utils = types.ModuleType("eval_suite.video_utils") +fake_video_utils.evaluate_video_chunk_new = Mock() +sys.modules["eval_suite.video_utils"] = fake_video_utils + +fake_image_utils = types.ModuleType("eval_suite.image_utils") +fake_image_utils.evaluate_sampled_images = Mock() +sys.modules["eval_suite.image_utils"] = fake_image_utils + +import evaluate + + +class MiniMaxSupportTest(unittest.TestCase): + def setUp(self): + fake_litellm.completion.reset_mock() + + def test_minimax_m3_formats_image_and_video_inputs(self): + model = LiteLLMWrapper( + model_name="minimax/MiniMax-M3", + use_langfuse=False, + ) + + result = model([ + {"type": "image", "content": "https://example.com/image.png"}, + {"type": "video", "content": "https://example.com/video.mp4"}, + ]) + + self.assertEqual(result, "ok") + messages = fake_litellm.completion.call_args.kwargs["messages"] + self.assertEqual( + messages[0]["content"][0], + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + }, + ) + self.assertEqual( + messages[1]["content"][0], + { + "type": "video_url", + "video_url": {"url": "https://example.com/video.mp4"}, + }, + ) + + def test_minimax_m27_rejects_media_inputs(self): + model = LiteLLMWrapper( + model_name="minimax/MiniMax-M2.7", + use_langfuse=False, + ) + + with self.assertRaisesRegex(ValueError, "only supports text input"): + model([{"type": "image", "content": "https://example.com/image.png"}]) + + fake_litellm.completion.assert_not_called() + + def test_minimax_m3_is_available_for_video_evaluation(self): + self.assertIn("minimax/MiniMax-M3", evaluate.VIDEO_MODELS) + self.assertNotIn("minimax/MiniMax-M2.7", evaluate.VIDEO_MODELS) + model = evaluate._create_video_model("minimax/MiniMax-M3") + + self.assertIsInstance(model, LiteLLMWrapper) + self.assertEqual(model.model_name, "minimax/MiniMax-M3") + + def test_existing_video_models_keep_their_wrapper(self): + model = evaluate._create_video_model("gemini/gemini-1.5-pro-002") + + self.assertIsInstance(model, FakeGeminiWrapper) + self.assertEqual(model.model_name, "gemini/gemini-1.5-pro-002") + + +if __name__ == "__main__": + unittest.main()