diff --git a/.github/ISSUE_TEMPLATE/llamacpp.yaml b/.github/ISSUE_TEMPLATE/llamacpp.yaml new file mode 100644 index 0000000..c5e370d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/llamacpp.yaml @@ -0,0 +1,78 @@ +name: "llamacpp issue" +description: 创建新功能请求 | Create a new ticket for a new feature request +title: "[llamacpp] - " +labels: [ + "question" +] +body: + - type: input + id: start_date + attributes: + label: "起始日期 | Start Date" + description: | + 起始开发日期 + Start of development + placeholder: "month/day/year" + validations: + required: false + - type: textarea + id: implementation_pr + attributes: + label: "实现PR | Implementation PR" + description: | + 实现该功能的Pull request + Pull request used + placeholder: "#Pull Request ID" + validations: + required: false + - type: textarea + id: reference_issues + attributes: + label: "相关Issues | Reference Issues" + description: | + 与该功能相关的issues + Common issues + placeholder: "#Issues IDs" + validations: + required: false + - type: textarea + id: summary + attributes: + label: "摘要 | Summary" + description: | + 简要描述新功能的特点 + Provide a brief explanation of the feature + placeholder: | + Describe in a few lines your feature request + validations: + required: true + - type: textarea + id: basic_example + attributes: + label: "基本示例 | Basic Example" + description: Indicate here some basic examples of your feature. + placeholder: A few specific words about your feature request. + validations: + required: true + - type: textarea + id: drawbacks + attributes: + label: "缺陷 | Drawbacks" + description: | + 该新功能有哪些缺陷/可能造成哪些影响？ + What are the drawbacks/impacts of your feature request ? + placeholder: | + Identify the drawbacks and impacts while being neutral on your feature request + validations: + required: true + - type: textarea + id: unresolved_question + attributes: + label: "未解决问题 | Unresolved questions" + description: | + 有哪些尚未解决的问题？ + What questions still remain unresolved ? + placeholder: | + Identify any unresolved issues. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/ollama.yaml b/.github/ISSUE_TEMPLATE/ollama.yaml new file mode 100644 index 0000000..e640151 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/ollama.yaml @@ -0,0 +1,78 @@ +name: "ollama issue" +description: 创建新功能请求 | Create a new ticket for a new feature request +title: "[ollama] - <title>" +labels: [ + "question" +] +body: + - type: input + id: start_date + attributes: + label: "起始日期 | Start Date" + description: | + 起始开发日期 + Start of development + placeholder: "month/day/year" + validations: + required: false + - type: textarea + id: implementation_pr + attributes: + label: "实现PR | Implementation PR" + description: | + 实现该功能的Pull request + Pull request used + placeholder: "#Pull Request ID" + validations: + required: false + - type: textarea + id: reference_issues + attributes: + label: "相关Issues | Reference Issues" + description: | + 与该功能相关的issues + Common issues + placeholder: "#Issues IDs" + validations: + required: false + - type: textarea + id: summary + attributes: + label: "摘要 | Summary" + description: | + 简要描述新功能的特点 + Provide a brief explanation of the feature + placeholder: | + Describe in a few lines your feature request + validations: + required: true + - type: textarea + id: basic_example + attributes: + label: "基本示例 | Basic Example" + description: Indicate here some basic examples of your feature. + placeholder: A few specific words about your feature request. + validations: + required: true + - type: textarea + id: drawbacks + attributes: + label: "缺陷 | Drawbacks" + description: | + 该新功能有哪些缺陷/可能造成哪些影响？ + What are the drawbacks/impacts of your feature request ? + placeholder: | + Identify the drawbacks and impacts while being neutral on your feature request + validations: + required: true + - type: textarea + id: unresolved_question + attributes: + label: "未解决问题 | Unresolved questions" + description: | + 有哪些尚未解决的问题？ + What questions still remain unresolved ? + placeholder: | + Identify any unresolved issues. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/vllm.yaml b/.github/ISSUE_TEMPLATE/vllm.yaml new file mode 100644 index 0000000..74f98a1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/vllm.yaml @@ -0,0 +1,78 @@ +name: "vllm issue" +description: 创建新功能请求 | Create a new ticket for a new feature request +title: "[vllm] - <title>" +labels: [ + "question" +] +body: + - type: input + id: start_date + attributes: + label: "起始日期 | Start Date" + description: | + 起始开发日期 + Start of development + placeholder: "month/day/year" + validations: + required: false + - type: textarea + id: implementation_pr + attributes: + label: "实现PR | Implementation PR" + description: | + 实现该功能的Pull request + Pull request used + placeholder: "#Pull Request ID" + validations: + required: false + - type: textarea + id: reference_issues + attributes: + label: "相关Issues | Reference Issues" + description: | + 与该功能相关的issues + Common issues + placeholder: "#Issues IDs" + validations: + required: false + - type: textarea + id: summary + attributes: + label: "摘要 | Summary" + description: | + 简要描述新功能的特点 + Provide a brief explanation of the feature + placeholder: | + Describe in a few lines your feature request + validations: + required: true + - type: textarea + id: basic_example + attributes: + label: "基本示例 | Basic Example" + description: Indicate here some basic examples of your feature. + placeholder: A few specific words about your feature request. + validations: + required: true + - type: textarea + id: drawbacks + attributes: + label: "缺陷 | Drawbacks" + description: | + 该新功能有哪些缺陷/可能造成哪些影响？ + What are the drawbacks/impacts of your feature request ? + placeholder: | + Identify the drawbacks and impacts while being neutral on your feature request + validations: + required: true + - type: textarea + id: unresolved_question + attributes: + label: "未解决问题 | Unresolved questions" + description: | + 有哪些尚未解决的问题？ + What questions still remain unresolved ? + placeholder: | + Identify any unresolved issues. + validations: + required: false diff --git a/README.md b/README.md index 52951b0..7f4c20c 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Unofficial [MiniCPM-V](https://github.com/OpenBMB/MiniCPM-V) support for [ComfyUI](https://github.com/comfyanonymous/ComfyUI). ## Important Updates +- **2024/08/09:** Added support for MiniCPM-V 2.6 (16GB+ video memory required) - **2024/05/22:** Added support for MiniCPM-Llama3-V 2.5 (16GB+ video memory required) - **2024/04/18:** Added ComfyUI nodes and workflow examples diff --git a/README.md.bak b/README.md.bak index ee06699..8c6d607 100644 --- a/README.md.bak +++ b/README.md.bak @@ -2,7 +2,7 @@ <img src="./assets/minicpmv.png" width="300em" ></img> -**A GPT-4V Level Multimodal LLM on Your Phone** +**A GPT-4V Level MLLM for Single Image, Multi Image and Video on Your Phone** <strong>[中文](./README_zh.md) | English</strong> @@ -11,17 +11,16 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> <p align="center"> - MiniCPM-Llama3-V 2.5 <a href="https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5">🤖</a> | - MiniCPM-V 2.0 <a href="https://huggingface.co/openbmb/MiniCPM-V-2/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-V-2">🤖</a> | - <a href="https://openbmb.vercel.app/minicpm-v-2-en"> Technical Blog </a> + MiniCPM-V 2.6 <a href="https://huggingface.co/openbmb/MiniCPM-V-2_6">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6">🤖</a> | MiniCPM-Llama3-V 2.5 <a href="https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5">🤖</a> | + <a href=https://arxiv.org/abs/2408.01800>MiniCPM-Llama3-V 2.5 Technical Report</a> </p> </div> -**MiniCPM-V** is a series of end-side multimodal LLMs (MLLMs) designed for vision-language understanding. The models take image and text as inputs and provide high-quality text outputs. Since February 2024, we have released 4 versions of the model, aiming to achieve **strong performance and efficient deployment**. The most notable models in this series currently include: +**MiniCPM-V** is a series of end-side multimodal LLMs (MLLMs) designed for vision-language understanding. The models take image, video and text as inputs and provide high-quality text outputs. Since February 2024, we have released 5 versions of the model, aiming to achieve **strong performance and efficient deployment**. The most notable models in this series currently include: -- **MiniCPM-Llama3-V 2.5**: 🔥🔥🔥 The latest and most capable model in the MiniCPM-V series. With a total of 8B parameters, the model **surpasses proprietary models such as GPT-4V-1106, Gemini Pro, Qwen-VL-Max and Claude 3** in overall performance. Equipped with the enhanced OCR and instruction-following capability, the model can also support multimodal conversation for **over 30 languages** including English, Chinese, French, Spanish, German etc. With help of quantization, compilation optimizations, and several efficient inference techniques on CPUs and NPUs, MiniCPM-Llama3-V 2.5 can be **efficiently deployed on end-side devices**. +- **MiniCPM-V 2.6**: 🔥🔥🔥 The latest and most capable model in the MiniCPM-V series. With a total of 8B parameters, the model **surpasses GPT-4V in single image, multi-image and video understanding**. It outperforms **GPT-4o mini, Gemini 1.5 Pro and Claude 3.5 Sonnet** in single image understanding, and advances MiniCPM-Llama3-V 2.5's features such as strong OCR capability, trustworthy behavior, multilingual support, and end-side deployment. Due to its superior token density, MiniCPM-V 2.6 can for the first time support real-time video understanding on end-side devices such as iPad. - **MiniCPM-V 2.0**: The lightest model in the MiniCPM-V series. With 2B parameters, it surpasses larger models such as Yi-VL 34B, CogVLM-Chat 17B, and Qwen-VL-Chat 10B in overall performance. It can accept image inputs of any aspect ratio and up to 1.8 million pixels (e.g., 1344x1344), achieving comparable performance with Gemini Pro in understanding scene-text and matches GPT-4V in low hallucination rates. @@ -29,7 +28,9 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> ## News <!-- omit in toc --> #### 📌 Pinned -* [2024.07.19] MiniCPM-Llama3-V 2.5 supports vLLM now! See [here](#vllm). +* [2024.08.06] 🔥🔥🔥 We open-source MiniCPM-V 2.6, which outperforms GPT-4V on single image, multi-image and video understanding. It advances popular features of MiniCPM-Llama3-V 2.5, and can support real-time video understanding on iPad. Try it now! +* [2024.08.03] MiniCPM-Llama3-V 2.5 technical report is released! See [here](https://arxiv.org/abs/2408.01800). +* [2024.07.19] MiniCPM-Llama3-V 2.5 supports vLLM now! See [here](#inference-with-vllm). * [2024.05.28] 🚀🚀🚀 MiniCPM-Llama3-V 2.5 now fully supports its feature in llama.cpp and ollama! Please pull the latest code **of our provided forks** ([llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md), [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5)). GGUF models in various sizes are available [here](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf/tree/main). MiniCPM-Llama3-V 2.5 series is **not supported by the official repositories yet**, and we are working hard to merge PRs. Please stay tuned! * [2024.05.28] 💫 We now support LoRA fine-tuning for MiniCPM-Llama3-V 2.5, using only 2 V100 GPUs! See more statistics [here](https://github.com/OpenBMB/MiniCPM-V/tree/main/finetune#model-fine-tuning-memory-usage-statistics). * [2024.05.23] 🔍 We've released a comprehensive comparison between Phi-3-vision-128k-instruct and MiniCPM-Llama3-V 2.5, including benchmarks evaluations, multilingual capabilities, and inference efficiency 🌟📊🌍🚀. Click [here](./docs/compare_with_phi-3_vision.md) to view more details. @@ -37,11 +38,14 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> <br> +<details> +<summary>Click to view more news.</summary> + * [2024.06.03] Now, you can run MiniCPM-Llama3-V 2.5 on multiple low VRAM GPUs(12 GB or 16 GB) by distributing the model's layers across multiple GPUs. For more details, Check this [link](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md). * [2024.05.25] MiniCPM-Llama3-V 2.5 now supports streaming outputs and customized system prompts. Try it [here](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5#usage)! * [2024.05.24] We release the MiniCPM-Llama3-V 2.5 [gguf](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf), which supports [llama.cpp](#inference-with-llamacpp) inference and provides a 6~8 token/s smooth decoding on mobile phones. Try it now! * [2024.05.20] We open-soure MiniCPM-Llama3-V 2.5, it has improved OCR capability and supports 30+ languages, representing the first end-side MLLM achieving GPT-4V level performance! We provide [efficient inference](#deployment-on-mobile-phone) and [simple fine-tuning](./finetune/readme.md). Try it now! -* [2024.04.23] MiniCPM-V-2.0 supports vLLM now! Click [here](#vllm) to view more details. +* [2024.04.23] MiniCPM-V-2.0 supports vLLM now! Click [here](#inference-with-vllm) to view more details. * [2024.04.18] We create a HuggingFace Space to host the demo of MiniCPM-V 2.0 at [here](https://huggingface.co/spaces/openbmb/MiniCPM-V-2)! * [2024.04.17] MiniCPM-V-2.0 supports deploying [WebUI Demo](#webui-demo) now! * [2024.04.15] MiniCPM-V-2.0 now also supports [fine-tuning](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v-2最佳实践.md) with the SWIFT framework! @@ -49,29 +53,803 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> * [2024.03.14] MiniCPM-V now supports [fine-tuning](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v最佳实践.md) with the SWIFT framework. Thanks to [Jintao](https://github.com/Jintao-Huang) for the contribution！ * [2024.03.01] MiniCPM-V now can be deployed on Mac! * [2024.02.01] We open-source MiniCPM-V and OmniLMM-12B, which support efficient end-side deployment and powerful multimodal capabilities correspondingly. +</details> ## Contents <!-- omit in toc --> +- [MiniCPM-V 2.6](#minicpm-v-26) - [MiniCPM-Llama3-V 2.5](#minicpm-llama3-v-25) - [MiniCPM-V 2.0](#minicpm-v-20) -- [Chat with Our Demo on Gradio](#chat-with-our-demo-on-gradio) +- [Chat with Our Demo on Gradio 🤗](#chat-with-our-demo-on-gradio-) - [Install](#install) - [Inference](#inference) - [Model Zoo](#model-zoo) - [Multi-turn Conversation](#multi-turn-conversation) + - [Chat with multiple images](#chat-with-multiple-images) + - [In-context few-shot learning](#in-context-few-shot-learning) + - [Chat with video](#chat-with-video) + - [Inference on Multiple GPUs](#inference-on-multiple-gpus) - [Inference on Mac](#inference-on-mac) - [Deployment on Mobile Phone](#deployment-on-mobile-phone) - [Inference with llama.cpp](#inference-with-llamacpp) + - [Inference with ollama](#inference-with-ollama) - [Inference with vLLM](#inference-with-vllm) - [Fine-tuning](#fine-tuning) -- [TODO](#todo) -- [🌟 Star History](#-star-history) -- [Citation](#citation) +- [FAQs](#faqs) + + +## MiniCPM-V 2.6 + +**MiniCPM-V 2.6** is the latest and most capable model in the MiniCPM-V series. The model is built on SigLip-400M and Qwen2-7B with a total of 8B parameters. It exhibits a significant performance improvement over MiniCPM-Llama3-V 2.5, and introduces new features for multi-image and video understanding. Notable features of MiniCPM-V 2.6 include: + +- 🔥 **Leading Performance.** + MiniCPM-V 2.6 achieves an average score of 65.2 on the latest version of OpenCompass, a comprehensive evaluation over 8 popular benchmarks. **With only 8B parameters, it surpasses widely used proprietary models like GPT-4o mini, GPT-4V, Gemini 1.5 Pro, and Claude 3.5 Sonnet** for single image understanding. + +- 🖼️ **Multi Image Understanding and In-context Learning.** MiniCPM-V 2.6 can also perform **conversation and reasoning over multiple images**. It achieves **state-of-the-art performance** on popular multi-image benchmarks such as Mantis-Eval, BLINK, Mathverse mv and Sciverse mv, and also shows promising in-context learning capability. + +- 🎬 **Video Understanding.** MiniCPM-V 2.6 can also **accept video inputs**, performing conversation and providing dense captions for spatial-temporal information. It outperforms **GPT-4V, Claude 3.5 Sonnet and LLaVA-NeXT-Video-34B** on Video-MME with/without subtitles. + +- 💪 **Strong OCR Capability and Others.** + MiniCPM-V 2.6 can process images with any aspect ratio and up to 1.8 million pixels (e.g., 1344x1344). It achieves **state-of-the-art performance on OCRBench, surpassing proprietary models such as GPT-4o, GPT-4V, and Gemini 1.5 Pro**. + Based on the the latest [RLAIF-V](https://github.com/RLHF-V/RLAIF-V/) and [VisCPM](https://github.com/OpenBMB/VisCPM) techniques, it features **trustworthy behaviors**, with significantly lower hallucination rates than GPT-4o and GPT-4V on Object HalBench, and supports **multilingual capabilities** on English, Chinese, German, French, Italian, Korean, etc. + + +- 🚀 **Superior Efficiency.** + In addition to its friendly size, MiniCPM-V 2.6 also shows **state-of-the-art token density** (i.e., number of pixels encoded into each visual token). **It produces only 640 tokens when processing a 1.8M pixel image, which is 75% fewer than most models**. This directly improves the inference speed, first-token latency, memory usage, and power consumption. As a result, MiniCPM-V 2.6 can efficiently support **real-time video understanding** on end-side devices such as iPad. + +- 💫 **Easy Usage.** +MiniCPM-V 2.6 can be easily used in various ways: (1) [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpmv-main/examples/llava/README-minicpmv2.6.md) and [ollama](https://github.com/OpenBMB/ollama/blob/minicpm-v2.6/examples/minicpm-v2.6/README.md) support for efficient CPU inference on local devices, (2) [int4](https://huggingface.co/openbmb/MiniCPM-V-2_6-int4) and [GGUF](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) format quantized models in 16 sizes, (3) [vLLM](#inference-with-vllm) support for high-throughput and memory-efficient inference, (4) fine-tuning on new domains and tasks, (5) quick local WebUI demo setup with [Gradio](#chat-with-our-demo-on-gradio), and (6) online web [demo](https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6). + +### Evaluation <!-- omit in toc --> +<div align="center"> + <img src=assets/radar_final.png width=66% /> +</div> + +<details> +<summary>Click to view single image results on OpenCompass, MME, MMVet, OCRBench, MMMU, MathVista, MMB, AI2D, TextVQA, DocVQA, HallusionBench, Object HalBench. </summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Token Density<sup>+</sup></th> + <th>OpenCompass</th> + <th>MME</th> + <th>MMVet</th> + <th>OCRBench</th> + <th>MMMU val</th> + <th>MathVista mini</th> + <th>MMB1.1 test</th> + <th>AI2D</th> + <th>TextVQA val</th> + <th>DocVQA test</th> + <th>HallusionBench</th> + <th>Object HalBench</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="15" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4o</td> + <td>-</td> + <td>1088</td> + <td>69.9</td> + <td>2328.7</td> + <td>69.1</td> + <td>736</td> + <td>69.2</td> + <td>61.3</td> + <td>82.2</td> + <td>84.6</td> + <td>-</td> + <td>92.8</td> + <td>55.0</td> + <td>17.6</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Claude 3.5 Sonnet</td> + <td>-</td> + <td>750</td> + <td>67.9</td> + <td>1920.0</td> + <td>66.0</td> + <td>788</td> + <td>65.9</td> + <td>61.6</td> + <td>78.5</td> + <td>80.2</td> + <td>-</td> + <td>95.2</td> + <td>49.9</td> + <td>13.8</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Gemini 1.5 Pro</td> + <td>-</td> + <td>-</td> + <td>64.4</td> + <td>2110.6</td> + <td>64.0</td> + <td>754</td> + <td>60.6</td> + <td>57.7</td> + <td>73.9</td> + <td>79.1</td> + <td>73.5</td> + <td>86.5</td> + <td>45.6</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4o mini</td> + <td>-</td> + <td>1088</td> + <td>64.1</td> + <td>2003.4</td> + <td>66.9</td> + <td>785</td> + <td>60.0</td> + <td>52.4</td> + <td>76.0</td> + <td>77.8</td> + <td>-</td> + <td>-</td> + <td>46.1</td> + <td>12.4</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>1088</td> + <td>63.5</td> + <td>2070.2</td> + <td>67.5</td> + <td>656</td> + <td>61.7</td> + <td>54.7</td> + <td>79.8</td> + <td>78.6</td> + <td>78.0</td> + <td>87.2</td> + <td>43.9</td> + <td>14.2</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Step-1V</td> + <td>-</td> + <td>-</td> + <td>59.5</td> + <td>2206.4</td> + <td>63.3</td> + <td>625</td> + <td>49.9</td> + <td>44.8</td> + <td>78.0</td> + <td>79.2</td> + <td>71.6</td> + <td>-</td> + <td>48.4</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Qwen-VL-Max</td> + <td>-</td> + <td>784</td> + <td>58.3</td> + <td>2281.7</td> + <td>61.8</td> + <td>684</td> + <td>52.0</td> + <td>43.4</td> + <td>74.6</td> + <td>75.7</td> + <td>79.5</td> + <td>93.1</td> + <td>41.2</td> + <td>13.4</td> + </tr> + <tr> + <td colspan="15" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Yi-34B</td> + <td>34B</td> + <td>157</td> + <td>55.0</td> + <td>2006.5</td> + <td>50.7</td> + <td>574</td> + <td>48.8</td> + <td>40.4</td> + <td>77.8</td> + <td>78.9</td> + <td>69.3</td> + <td>-</td> + <td>34.8</td> + <td>12.6</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Mini-Gemini-HD-34B</td> + <td>34B</td> + <td>157</td> + <td>-</td> + <td>2141.0</td> + <td>59.3</td> + <td>518</td> + <td>48.0</td> + <td>43.3</td> + <td>-</td> + <td>80.5</td> + <td>74.1</td> + <td>78.9</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Cambrian-34B</td> + <td>34B</td> + <td>1820</td> + <td>58.3</td> + <td>2049.9</td> + <td>53.2</td> + <td>591</td> + <td>50.4</td> + <td>50.3</td> + <td>77.8</td> + <td>79.5</td> + <td>76.7</td> + <td>75.5</td> + <td>41.6</td> + <td>14.7</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GLM-4V-9B</td> + <td>13B</td> + <td>784</td> + <td>59.1</td> + <td>2018.8</td> + <td>58.0</td> + <td>776</td> + <td>46.9</td> + <td>51.1</td> + <td>67.9</td> + <td>71.2</td> + <td>-</td> + <td>-</td> + <td>45.0</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>706</td> + <td>64.1</td> + <td>2215.1</td> + <td>54.3</td> + <td>794</td> + <td><strong>51.2</strong></td> + <td>58.3</td> + <td><strong>79.4</strong></td> + <td><strong>83.6</strong></td> + <td>77.4</td> + <td><strong>91.6</strong></td> + <td>45.0</td> + <td>21.3</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">MiniCPM-Llama-V 2.5</td> + <td>8B</td> + <td>1882</td> + <td>58.8</td> + <td>2024.6</td> + <td>52.8</td> + <td>725</td> + <td>45.8</td> + <td>54.3</td> + <td>72.0</td> + <td>78.4</td> + <td>76.6</td> + <td>84.8</td> + <td>42.4</td> + <td>10.3</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>2822</strong></td> + <td><strong>65.2</strong></td> + <td><strong>2348.4</strong>*</td> + <td><strong>60.0</strong></td> + <td><strong>852</strong>*</td> + <td>49.8*</td> + <td><strong>60.6</strong></td> + <td>78.0</td> + <td>82.1</td> + <td><strong>80.1<strong></td> + <td>90.8</td> + <td><strong>48.1</strong>*</td> + <td><strong>8.2</strong></td> + </tr> + </tbody> +</table> + +</div> +* We evaluate this benchmark using chain-of-thought prompting. Specifically, for MME, we used this technique only for the Cognition set. + +<sup>+</sup> Token Density: number of pixels encoded into each visual token at maximum resolution, i.e., # pixels at maximum resolution / # visual tokens. + +Note: For proprietary models, we calculate token density based on the image encoding charging strategy defined in the official API documentation, which provides an upper-bound estimation. + +</details> + + +<details> +<summary>Click to view multi-image results on Mantis Eval, BLINK, Mathverse mv, Sciverse mv, MIRB.</summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Mantis Eval</th> + <th>BLINK val</th> + <th>Mathverse mv</th> + <th>Sciverse mv</th> + <th>MIRB</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="7" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>62.7</td> + <td>54.6</td> + <td>60.3</td> + <td>66.9</td> + <td>53.1</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Interleave-14B</td> + <td>14B</td> + <td>66.4</td> + <td>52.6</td> + <td>32.7</td> + <td>30.2</td> + <td>-</td> + </tr> + <tr> + <td colspan="7" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Emu2-Chat</td> + <td>37B</td> + <td>37.8</td> + <td>36.2</td> + <td>-</td> + <td>27.2</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">CogVLM</td> + <td>17B</td> + <td>45.2</td> + <td>41.1</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">VPG-C</td> + <td>7B</td> + <td>52.4</td> + <td>43.1</td> + <td>24.3</td> + <td>23.1</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">VILA 8B</td> + <td>8B</td> + <td>51.2</td> + <td>39.3</td> + <td>-</td> + <td>36.5</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternLM-XComposer-2.5</td> + <td>8B</td> + <td>53.1*</td> + <td>48.9</td> + <td>32.1*</td> + <td>-</td> + <td>42.5</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>59.0*</td> + <td>50.9</td> + <td>30.5*</td> + <td>34.4*</td> + <td><strong>56.9*</strong></td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>69.1</strong></td> + <td><strong>53.0</strong></td> + <td><strong>84.9</strong></td> + <td><strong>74.9</strong></td> + <td>53.8</td> + </tr> + </tbody> +</table> + +</div> +* We evaluate the officially released checkpoint by ourselves. +</details> + +<details> +<summary>Click to view video results on Video-MME and Video-ChatGPT.</summary> +<div align="center"> +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th colspan="2">Video-MME</th> + <th colspan="5">Video-ChatGPT</th> + </tr> + <tr> + <th align="left"></th> + <th></th> + <th>w/o subs</th> + <th>w subs</th> + <th>Correctness</th> + <th>Detail</th> + <th>Context</th> + <th>Temporal</th> + <th>Consistency</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="9" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Claude 3.5 Sonnet</td> + <td>-</td> + <td>60.0</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>59.9</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td colspan="9" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-7B</td> + <td>7B</td> + <td>-</td> + <td>-</td> + <td>3.39</td> + <td>3.29</td> + <td>3.92</td> + <td>2.60</td> + <td>3.12</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-34B</td> + <td>34B</td> + <td>-</td> + <td>-</td> + <td>3.29</td> + <td>3.23</td> + <td>3.83</td> + <td>2.51</td> + <td>3.47</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">CogVLM2-Video</td> + <td>12B</td> + <td>-</td> + <td>-</td> + <td>3.49</td> + <td><strong>3.46</strong></td> + <td>3.23</td> + <td><strong>2.98</strong></td> + <td><strong>3.64</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LongVA</td> + <td>7B</td> + <td>52.4</td> + <td>54.3</td> + <td>3.05</td> + <td>3.09</td> + <td>3.77</td> + <td>2.44</td> + <td><strong>3.64</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>54.0</td> + <td>56.9</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternLM-XComposer-2.5</td> + <td>8B</td> + <td>55.8</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Video</td> + <td>32B</td> + <td>60.2</td> + <td>63.0</td> + <td>3.48</td> + <td>3.37</td> + <td><strong>3.95</strong></td> + <td>2.64</td> + <td>3.28</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>60.9</strong></td> + <td><strong>63.6</strong></td> + <td><strong>3.59</strong></td> + <td>3.28</td> + <td>3.93</td> + <td>2.73</td> + <td>3.62</td> + </tr> + </tbody> +</table> +</div> +</details> + + +<details> +<summary>Click to view few-shot results on TextVQA, VizWiz, VQAv2, OK-VQA.</summary> +<div align="center"> +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Shot</th> + <th>TextVQA val</th> + <th>VizWiz test-dev</th> + <th>VQAv2 test-dev</th> + <th>OK-VQA val</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">Flamingo</td> + <td rowspan="3">80B</td> + <td>0*</td> + <td>35.0</td> + <td>31.6</td> + <td>56.3</td> + <td>40.6</td> + </tr> + <tr> + <td>4</td> + <td>36.5</td> + <td>39.6</td> + <td>63.1</td> + <td><strong>57.4</strong></td> + </tr> + <tr> + <td>8</td> + <td>37.3</td> + <td>44.8</td> + <td>65.6</td> + <td>57.5</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">IDEFICS</td> + <td rowspan="3">80B</td> + <td>0*</td> + <td>30.9</td> + <td>36.0</td> + <td>60.0</td> + <td>45.2</td> + </tr> + <tr> + <td>4</td> + <td>34.3</td> + <td>40.4</td> + <td>63.6</td> + <td>52.4</td> + </tr> + <tr> + <td>8</td> + <td>35.7</td> + <td>46.1</td> + <td>64.8</td> + <td>55.1</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">OmniCorpus</td> + <td rowspan="3">7B</td> + <td>0*</td> + <td>43.0</td> + <td>49.8</td> + <td>63.2</td> + <td>45.5</td> + </tr> + <tr> + <td>4</td> + <td>45.4</td> + <td>51.3</td> + <td>64.5</td> + <td>46.5</td> + </tr> + <tr> + <td>8</td> + <td>45.6</td> + <td>52.2</td> + <td>64.7</td> + <td>46.6</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">Emu2</td> + <td rowspan="3">37B</td> + <td>0</td> + <td>26.4</td> + <td>40.4</td> + <td>33.5</td> + <td>26.7</td> + </tr> + <tr> + <td>4</td> + <td>48.2</td> + <td>54.6</td> + <td>67.0</td> + <td>53.2</td> + </tr> + <tr> + <td>8</td> + <td>49.3</td> + <td>54.7</td> + <td>67.8</td> + <td>54.1</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="2">MM1</td> + <td rowspan="2">30B</td> + <td>0</td> + <td>26.2</td> + <td>40.4</td> + <td>48.9</td> + <td>26.7</td> + </tr> + <tr> + <td>8</td> + <td>49.3</td> + <td>54.7</td> + <td><strong>70.9</strong></td> + <td>54.1</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td align="left" nowrap="nowrap" rowspan="3">MiniCPM-V 2.6<sup>+</sup></td> + <td rowspan="3">8B</td> + <td>0</td> + <td>43.9</td> + <td>33.8</td> + <td>45.4</td> + <td>23.9</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td>4</td> + <td>63.6</td> + <td>60.5</td> + <td>65.5</td> + <td>50.1</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td>8</td> + <td><strong>64.6</strong></td> + <td><strong>63.4</strong></td> + <td>68.2</td> + <td>51.4</td> + </tr> + </tbody> +</table> + + +</div> +* denotes zero image shot and two additional text shots following Flamingo. + +<sup>+</sup> We evaluate the pretraining ckpt without SFT. +</details> + +### Examples <!-- omit in toc --> + +<div style="display: flex; flex-direction: column; align-items: center;"> + <img src="assets/minicpmv2_6/multi_img-bike.png" alt="Bike" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multi_img-menu.png" alt="Menu" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multi_img-code.png" alt="Code" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/ICL-Mem.png" alt="Mem" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multiling-medal.png" alt="medal" style="margin-bottom: 10px;"> +</div> +<details> + <summary>Click to view more cases.</summary> + <div style="display: flex; flex-direction: column; align-items: center;"> + <img src="assets/minicpmv2_6/ICL-elec.png" alt="elec" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multiling-olympic.png" alt="Menu" style="margin-bottom: 10px;"> + </div> +</details> + +We deploy MiniCPM-V 2.6 on end devices. The demo video is the raw screen recording on a iPad Pro without edition. + +<table align="center"> + <p align="center"> + <img src="assets/gif_cases/ai.gif" width=32%/> +      + <img src="assets/gif_cases/beer.gif" width=32%/> + </p> +</table> + +<table align="center"> + <p align="center"> + <img src="assets/gif_cases/ticket.gif" width=32%/> +      + <img src="assets/gif_cases/wfh.gif" width=32%/> + </p> +</table> + +<table align="center"> + <p align="center"> + <video src="https://github.com/user-attachments/assets/21f4b818-ede1-4822-920e-91281725c830" width="360" /> </video> + <!-- <video src="https://github.com/user-attachments/assets/c835f757-206b-4d9c-8e36-70d67b453628" width="360" /> </video> --> + </p> +</table> ## MiniCPM-Llama3-V 2.5 +<details> +<summary>Click to view more details of MiniCPM-Llama3-V 2.5</summary> + **MiniCPM-Llama3-V 2.5** is the latest model in the MiniCPM-V series. The model is built on SigLip-400M and Llama3-8B-Instruct with a total of 8B parameters. It exhibits a significant performance improvement over MiniCPM-V 2.0. Notable features of MiniCPM-Llama3-V 2.5 include: - 🔥 **Leading Performance.** @@ -92,15 +870,6 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> - 💫 **Easy Usage.** MiniCPM-Llama3-V 2.5 can be easily used in various ways: (1) [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) and [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5) support for efficient CPU inference on local devices, (2) [GGUF](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf) format quantized models in 16 sizes, (3) efficient [LoRA](https://github.com/OpenBMB/MiniCPM-V/tree/main/finetune#lora-finetuning) fine-tuning with only 2 V100 GPUs, (4) [streaming output](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5#usage), (5) quick local WebUI demo setup with [Gradio](https://github.com/OpenBMB/MiniCPM-V/blob/main/web_demo_2.5.py) and [Streamlit](https://github.com/OpenBMB/MiniCPM-V/blob/main/web_demo_streamlit-2_5.py), and (6) interactive demos on [HuggingFace Spaces](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5). -### MiniCPM-Llama3-V 2.5 Common Module Navigation <!-- omit in toc --> -You can click links in the following table to quickly access the commonly used functions for MiniCPM-Llama3-V 2.5. -| Functional Categories | | | | | | | || -|:--------:|:------:|:--------------:|:--------:|:-------:|:-----------:|:-----------:|:--------:|:-----------:| -| Inference | [Transformers](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md) | [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5) | [SWIFT](./docs/swift_train_and_infer.md) | [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) | [Xinfrence](./docs/xinference_infer.md) | [Gradio](./web_demo_2.5.py) | [Streamlit](./web_demo_streamlit-2_5.py) |[vLLM](#vllm) -| Finetune | [Full-parameter](./finetune/readme.md) | [Lora](./finetune/readme.md) | [SWIFT](./docs/swift_train_and_infer.md) | | | | | | -| Edge Deployment | [apk](http://minicpm.modelbest.cn/android/modelbest-release-20240528_182155.apk) | [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) | | | | | | | -| Quantize | [Bnb](./quantize/bnb_quantize.py) | - ### Evaluation <!-- omit in toc --> <div align="center"> @@ -399,20 +1168,8 @@ You can click links in the following table to quickly access the commonly used f </p> </table> -We deploy MiniCPM-Llama3-V 2.5 on end devices. The demo video is the raw screen recording on a Xiaomi 14 Pro without edition. +</details> -<table align="center"> - <p align="center"> - <img src="assets/gif_cases/ticket.gif" width=32%/> - <img src="assets/gif_cases/meal_plan.gif" width=32%/> - </p> -</table> - -<table align="center"> - <p align="center"> - <img src="assets/gif_cases/1-4.gif" width=64%/> - </p> -</table> ## MiniCPM-V 2.0 @@ -469,14 +1226,14 @@ We deploy MiniCPM-V 2.0 on end devices. The demo video is the raw screen recordi | OmniLMM-12B | [Document](./omnilmm_en.md) | -## Chat with Our Demo on Gradio +## Chat with Our Demo on Gradio 🤗 -We provide online and local demos powered by HuggingFace [Gradio](https://github.com/gradio-app/gradio), the most popular model deployment framework nowadays. It supports streaming outputs, progress bars, queuing, alerts, and other useful features. +We provide online and local demos powered by Hugging Face Gradio <a href='https://github.com/gradio-app/gradio'><img src='https://img.shields.io/github/stars/gradio-app/gradio'></a>, the most popular model deployment framework nowadays. It supports streaming outputs, progress bars, queuing, alerts, and other useful features. ### Online Demo <!-- omit in toc --> -Click here to try out the online demo of [MiniCPM-Llama3-V 2.5](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) ｜ [MiniCPM-V 2.0](https://huggingface.co/spaces/openbmb/MiniCPM-V-2) on HuggingFace Spaces. +Click here to try out the online demo of [MiniCPM-V 2.6](https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6) | [MiniCPM-Llama3-V 2.5](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) | [MiniCPM-V 2.0](https://huggingface.co/spaces/openbmb/MiniCPM-V-2). ### Local WebUI Demo <!-- omit in toc --> @@ -488,10 +1245,8 @@ pip install -r requirements.txt ```shell # For NVIDIA GPUs, run: -python web_demo_2.5.py --device cuda +python web_demo_2.6.py --device cuda -# For Mac with MPS (Apple silicon or AMD GPUs), run: -PYTORCH_ENABLE_MPS_FALLBACK=1 python web_demo_2.5.py --device mps ``` @@ -524,9 +1279,12 @@ pip install -r requirements.txt | Model | Device | Memory |          Description | Download | |:-----------|:--:|:-----------:|:-------------------|:---------------:| -| MiniCPM-Llama3-V 2.5 | GPU | 19 GB | The lastest version, achieving state-of-the end-side multimodal performance. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5) | -| MiniCPM-Llama3-V 2.5 gguf | CPU | 5 GB | The gguf version, lower memory usage and faster inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)   [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-gguf) | -| MiniCPM-Llama3-V 2.5 int4 | GPU | 8 GB | The int4 quantized version，lower GPU memory usage. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-int4/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-int4) | +| MiniCPM-V 2.6| GPU | 17 GB | The latest version, achieving state-of-the-art end-side performance for single image, multi-image and video understanding. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6) | +| MiniCPM-V 2.6 gguf | CPU | 6 GB | The gguf version, lower memory usage and faster inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6-gguf) | +| MiniCPM-V 2.6 int4 | GPU | 7 GB | The int4 quantized version, lower GPU memory usage. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6-int4)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6-int4) | +| MiniCPM-Llama3-V 2.5 | GPU | 19 GB | Strong end-side multimodal performance. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5) | +| MiniCPM-Llama3-V 2.5 gguf | CPU | 6 GB | The gguf version, lower memory usage and faster inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)   [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-gguf) | +| MiniCPM-Llama3-V 2.5 int4 | GPU | 8 GB | The int4 quantized version, lower GPU memory usage. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-int4/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-int4) | | MiniCPM-V 2.0 | GPU | 8 GB | Light version, balance the performance the computation cost. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2) | | MiniCPM-V 1.0 | GPU | 7 GB | Lightest version, achieving the fastest inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-V)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V) | @@ -540,30 +1298,40 @@ Please refer to the following codes to run. ```python -from chat import MiniCPMVChat, img2base64 import torch -import json +from PIL import Image +from transformers import AutoModel, AutoTokenizer torch.manual_seed(0) -chat_model = MiniCPMVChat('openbmb/MiniCPM-Llama3-V-2_5') +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) -im_64 = img2base64('./assets/airplane.jpeg') +image = Image.open('./assets/airplane.jpeg').convert('RGB') # First round chat -msgs = [{"role": "user", "content": "Tell me the model of this aircraft."}] +question = "Tell me the model of this aircraft." +msgs = [{'role': 'user', 'content': [image, question]}] -inputs = {"image": im_64, "question": json.dumps(msgs)} -answer = chat_model.chat(inputs) +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) print(answer) # Second round chat # pass history context of multi-turn conversation -msgs.append({"role": "assistant", "content": answer}) -msgs.append({"role": "user", "content": "Introduce something about Airbus A380."}) +msgs.append({"role": "assistant", "content": [answer]}) +msgs.append({"role": "user", "content": ["Introduce something about Airbus A380."]}) -inputs = {"image": im_64, "question": json.dumps(msgs)} -answer = chat_model.chat(inputs) +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) print(answer) ``` @@ -575,6 +1343,127 @@ You will get the following output: "The Airbus A380 is a double-deck, wide-body, four-engine jet airliner made by Airbus. It is the world's largest passenger airliner and is known for its long-haul capabilities. The aircraft was developed to improve efficiency and comfort for passengers traveling over long distances. It has two full-length passenger decks, which can accommodate more passengers than a typical single-aisle airplane. The A380 has been operated by airlines such as Lufthansa, Singapore Airlines, and Emirates, among others. It is widely recognized for its unique design and significant impact on the aviation industry." ``` +#### Chat with multiple images +<details> +<summary> Click to view Python code running MiniCPM-V 2.6 with multiple images input. </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +image1 = Image.open('image1.jpg').convert('RGB') +image2 = Image.open('image2.jpg').convert('RGB') +question = 'Compare image 1 and image 2, tell me about the differences between image 1 and image 2.' + +msgs = [{'role': 'user', 'content': [image1, image2, question]}] + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) +print(answer) +``` +</details> + +#### In-context few-shot learning +<details> +<summary> Click to view Python code running MiniCPM-V 2.6 with few-shot input. </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +question = "production date" +image1 = Image.open('example1.jpg').convert('RGB') +answer1 = "2023.08.04" +image2 = Image.open('example2.jpg').convert('RGB') +answer2 = "2007.04.24" +image_test = Image.open('test.jpg').convert('RGB') + +msgs = [ + {'role': 'user', 'content': [image1, question]}, {'role': 'assistant', 'content': [answer1]}, + {'role': 'user', 'content': [image2, question]}, {'role': 'assistant', 'content': [answer2]}, + {'role': 'user', 'content': [image_test, question]} +] + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) +print(answer) +``` +</details> + +#### Chat with video +<details> +<summary> Click to view Python code running MiniCPM-V 2.6 with video input. </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer +from decord import VideoReader, cpu # pip install decord + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +MAX_NUM_FRAMES=64 # if cuda OOM set a smaller number + +def encode_video(video_path): + def uniform_sample(l, n): + gap = len(l) / n + idxs = [int(i * gap + gap / 2) for i in range(n)] + return [l[i] for i in idxs] + + vr = VideoReader(video_path, ctx=cpu(0)) + sample_fps = round(vr.get_avg_fps() / 1) # FPS + frame_idx = [i for i in range(0, len(vr), sample_fps)] + if len(frame_idx) > MAX_NUM_FRAMES: + frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES) + frames = vr.get_batch(frame_idx).asnumpy() + frames = [Image.fromarray(v.astype('uint8')) for v in frames] + print('num frames:', len(frames)) + return frames + +video_path="video_test.mp4" +frames = encode_video(video_path) +question = "Describe the video" +msgs = [ + {'role': 'user', 'content': frames + [question]}, +] + +# Set decode params for video +params = {} +params["use_image_id"] = False +params["max_slice_nums"] = 2 # use 1 if cuda OOM and video resolution > 448*448 + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + **params +) +print(answer) +``` +</details> + + ### Inference on Multiple GPUs You can run MiniCPM-Llama3-V 2.5 on multiple low VRAM GPUs (12 GB or 16 GB) by distributing the model's layers across multiple GPUs. Please refer to this [tutorial](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md) for detailed instructions on how to load the model and inference using multiple low VRAM GPUs. @@ -615,33 +1504,100 @@ PYTORCH_ENABLE_MPS_FALLBACK=1 python test.py </details> ### Deployment on Mobile Phone -MiniCPM-Llama3-V 2.5 and MiniCPM-V 2.0 can be deployed on mobile phones with Android operating systems. 🚀 Click [MiniCPM-Llama3-V 2.5](http://minicpm.modelbest.cn/android/modelbest-release-20240528_182155.apk) / [MiniCPM-V 2.0](https://github.com/OpenBMB/mlc-MiniCPM) to install apk. +MiniCPM-V 2.0 can be deployed on mobile phones with Android operating systems. 🚀 Click [MiniCPM-V 2.0](https://github.com/OpenBMB/mlc-MiniCPM) to install apk. -### Inference with llama.cpp<a id="inference-with-llamacpp"></a> -MiniCPM-Llama3-V 2.5 can run with llama.cpp now! See our fork of [llama.cpp](https://github.com/OpenBMB/llama.cpp/tree/minicpm-v2.5/examples/minicpmv) for more detail. This implementation supports smooth inference of 6~8 token/s on mobile phones (test environment：Xiaomi 14 pro + Snapdragon 8 Gen 3). +### Inference with llama.cpp +MiniCPM-V 2.6 can run with llama.cpp now! See [our fork of llama.cpp](https://github.com/OpenBMB/llama.cpp/tree/minicpmv-main/examples/llava/README-minicpmv2.6.md) for more detail. This implementation supports smooth inference of 16~18 token/s on iPad (test environment：iPad Pro + M4). -### Inference with vLLM<a id="vllm"></a> +### Inference with ollama +MiniCPM-V 2.6 can run with ollama now! See [our fork of ollama](https://github.com/OpenBMB/ollama/blob/minicpm-v2.6/examples/minicpm-v2.6/README.md) for more detail. This implementation supports smooth inference of 16~18 token/s on iPad (test environment：iPad Pro + M4). + +### Inference with vLLM <details> -<summary> vLLM now officially supports MiniCPM-V 2.0 and MiniCPM-Llama3-V 2.5, Click to see. </summary> +<summary> vLLM now officially supports MiniCPM-V 2.6, MiniCPM-Llama3-V 2.5 and MiniCPM-V 2.0, Click to see. </summary> -1. Clone the official vLLM: +1. Install vLLM(>=0.5.4): ```shell -git clone https://github.com/vllm-project/vllm.git +pip install vllm ``` -2. Install vLLM: -```shell -cd vllm -pip install -e . -``` -3. Install timm: +2. Install timm: (optional, MiniCPM-V 2.0 need timm) ```shell pip install timm==0.9.10 ``` -4. Run the example:（If you use model in local path, please update the model code to the latest version on Hugging Face.) -```shell -python examples/minicpmv_example.py +3. Run the example(for image): +```python +from transformers import AutoTokenizer +from PIL import Image +from vllm import LLM, SamplingParams + +MODEL_NAME = "openbmb/MiniCPM-V-2_6" +# Also available for previous models +# MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5" +# MODEL_NAME = "HwwwH/MiniCPM-V-2" + +image = Image.open("xxx.png").convert("RGB") +tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) +llm = LLM( + model=MODEL_NAME, + trust_remote_code=True, + gpu_memory_utilization=1, + max_model_len=2048 +) + +messages = [{ + "role": + "user", + "content": + # Number of images + "(<image>./</image>)" + \ + "\nWhat is the content of this image?" +}] +prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True +) + +# Single Inference +inputs = { + "prompt": prompt, + "multi_modal_data": { + "image": image + # Multi images, the number of images should be equal to that of `(<image>./</image>)` + # "image": [image, image] + }, +} +# Batch Inference +# inputs = [{ +# "prompt": prompt, +# "multi_modal_data": { +# "image": image +# }, +# } for _ in 2] + + +# 2.6 +stop_tokens = ['<|im_end|>', '<|endoftext|>'] +stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] +# 2.0 +# stop_token_ids = [tokenizer.eos_id] +# 2.5 +# stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id] + +sampling_params = SamplingParams( + stop_token_ids=stop_token_ids, + use_beam_search=True, + temperature=0, + best_of=3, + max_tokens=1024 +) + +outputs = llm.generate(inputs, sampling_params=sampling_params) + +print(outputs[0].outputs[0].text) ``` +4. click [here](https://modelbest.feishu.cn/wiki/C2BWw4ZP0iCDy7kkCPCcX2BHnOf?from=from_copylink) if you want to use it with *video*, or get more details about `vLLM`. </details> ## Fine-tuning @@ -658,12 +1614,8 @@ We now support MiniCPM-V series fine-tuning with the SWIFT framework. SWIFT supp Best Practices：[MiniCPM-V 1.0](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v最佳实践.md), [MiniCPM-V 2.0](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v-2最佳实践.md) - - -## TODO - -- [x] MiniCPM-V fine-tuning support -- [ ] Code release for real-time interactive assistant +## FAQs +Click here to view the [FAQs](./docs/faqs.md) ## Model License <!-- omit in toc --> @@ -678,7 +1630,7 @@ Best Practices：[MiniCPM-V 1.0](https://github.com/modelscope/swift/blob/main/d As LMMs, MiniCPM-V models (including OmniLMM) generate contents by learning a large amount of multimodal corpora, but they cannot comprehend, express personal opinions or make value judgement. Anything generated by MiniCPM-V models does not represent the views and positions of the model developers -We will not be liable for any problems arising from the use of MiniCPMV-V models, including but not limited to data security issues, risk of public opinion, or any risks and problems arising from the misdirection, misuse, dissemination or misuse of the model. +We will not be liable for any problems arising from the use of MiniCPM-V models, including but not limited to data security issues, risk of public opinion, or any risks and problems arising from the misdirection, misuse, dissemination or misuse of the model. ## Institutions <!-- omit in toc --> @@ -689,13 +1641,7 @@ This project is developed by the following institutions: - <img src="assets/modelbest.png" width="28px"> [ModelBest](https://modelbest.cn/) - <img src="assets/zhihu.webp" width="28px"> [Zhihu](https://www.zhihu.com/ ) -## Other Multimodal Projects from Our Team <!-- omit in toc --> - -👏 Welcome to explore other multimodal projects of our team: - -[VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) - -## 🌟 Star History +## 🌟 Star History <!-- omit in toc --> <table align="center"> @@ -723,33 +1669,22 @@ This project is developed by the following institutions: /> </picture> --> -## Citation +## Key Techniques and Other Multimodal Projects <!-- omit in toc --> + +👏 Welcome to explore key techniques of MiniCPM-V and other multimodal projects of our team: + +[VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) + + +## Citation <!-- omit in toc --> If you find our model/code/paper helpful, please consider cite our papers 📝 and star us ⭐️！ ```bib -@article{yu2023rlhf, - title={Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback}, - author={Yu, Tianyu and Yao, Yuan and Zhang, Haoye and He, Taiwen and Han, Yifeng and Cui, Ganqu and Hu, Jinyi and Liu, Zhiyuan and Zheng, Hai-Tao and Sun, Maosong and others}, - journal={arXiv preprint arXiv:2312.00849}, - year={2023} -} -@article{viscpm, - title={Large Multilingual Models Pivot Zero-Shot Multimodal Learning across Languages}, - author={Jinyi Hu and Yuan Yao and Chongyi Wang and Shan Wang and Yinxu Pan and Qianyu Chen and Tianyu Yu and Hanghao Wu and Yue Zhao and Haoye Zhang and Xu Han and Yankai Lin and Jiao Xue and Dahai Li and Zhiyuan Liu and Maosong Sun}, - journal={arXiv preprint arXiv:2308.12038}, - year={2023} -} -@article{xu2024llava-uhd, - title={{LLaVA-UHD}: an LMM Perceiving Any Aspect Ratio and High-Resolution Images}, - author={Xu, Ruyi and Yao, Yuan and Guo, Zonghao and Cui, Junbo and Ni, Zanlin and Ge, Chunjiang and Chua, Tat-Seng and Liu, Zhiyuan and Huang, Gao}, - journal={arXiv preprint arXiv:2403.11703}, - year={2024} -} -@article{yu2024rlaifv, - title={RLAIF-V: Aligning MLLMs through Open-Source AI Feedback for Super GPT-4V Trustworthiness}, - author={Yu, Tianyu and Zhang, Haoye and Yao, Yuan and Dang, Yunkai and Chen, Da and Lu, Xiaoman and Cui, Ganqu and He, Taiwen and Liu, Zhiyuan and Chua, Tat-Seng and Sun, Maosong}, - journal={arXiv preprint arXiv:2405.17220}, +@article{yao2024minicpm, + title={MiniCPM-V: A GPT-4V Level MLLM on Your Phone}, + author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others}, + journal={arXiv preprint arXiv:2408.01800}, year={2024} } ``` \ No newline at end of file diff --git a/README_en.md b/README_en.md index c51576f..446b92e 100644 --- a/README_en.md +++ b/README_en.md @@ -2,7 +2,7 @@ <img src="./assets/minicpmv.png" width="300em" ></img> -**A GPT-4V Level Multimodal LLM on Your Phone** +**A GPT-4V Level MLLM for Single Image, Multi Image and Video on Your Phone** <strong>[中文](./README_zh.md) | English</strong> @@ -11,17 +11,16 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> <p align="center"> - MiniCPM-Llama3-V 2.5 <a href="https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5">🤖</a> | - MiniCPM-V 2.0 <a href="https://huggingface.co/openbmb/MiniCPM-V-2/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-V-2">🤖</a> | - <a href="https://openbmb.vercel.app/minicpm-v-2-en"> Technical Blog </a> + MiniCPM-V 2.6 <a href="https://huggingface.co/openbmb/MiniCPM-V-2_6">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6">🤖</a> | MiniCPM-Llama3-V 2.5 <a href="https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5">🤖</a> | + <a href=https://arxiv.org/abs/2408.01800>MiniCPM-Llama3-V 2.5 Technical Report</a> </p> </div> -**MiniCPM-V** is a series of end-side multimodal LLMs (MLLMs) designed for vision-language understanding. The models take image and text as inputs and provide high-quality text outputs. Since February 2024, we have released 4 versions of the model, aiming to achieve **strong performance and efficient deployment**. The most notable models in this series currently include: +**MiniCPM-V** is a series of end-side multimodal LLMs (MLLMs) designed for vision-language understanding. The models take image, video and text as inputs and provide high-quality text outputs. Since February 2024, we have released 5 versions of the model, aiming to achieve **strong performance and efficient deployment**. The most notable models in this series currently include: -- **MiniCPM-Llama3-V 2.5**: 🔥🔥🔥 The latest and most capable model in the MiniCPM-V series. With a total of 8B parameters, the model **surpasses proprietary models such as GPT-4V-1106, Gemini Pro, Qwen-VL-Max and Claude 3** in overall performance. Equipped with the enhanced OCR and instruction-following capability, the model can also support multimodal conversation for **over 30 languages** including English, Chinese, French, Spanish, German etc. With help of quantization, compilation optimizations, and several efficient inference techniques on CPUs and NPUs, MiniCPM-Llama3-V 2.5 can be **efficiently deployed on end-side devices**. +- **MiniCPM-V 2.6**: 🔥🔥🔥 The latest and most capable model in the MiniCPM-V series. With a total of 8B parameters, the model **surpasses GPT-4V in single image, multi-image and video understanding**. It outperforms **GPT-4o mini, Gemini 1.5 Pro and Claude 3.5 Sonnet** in single image understanding, and advances MiniCPM-Llama3-V 2.5's features such as strong OCR capability, trustworthy behavior, multilingual support, and end-side deployment. Due to its superior token density, MiniCPM-V 2.6 can for the first time support real-time video understanding on end-side devices such as iPad. - **MiniCPM-V 2.0**: The lightest model in the MiniCPM-V series. With 2B parameters, it surpasses larger models such as Yi-VL 34B, CogVLM-Chat 17B, and Qwen-VL-Chat 10B in overall performance. It can accept image inputs of any aspect ratio and up to 1.8 million pixels (e.g., 1344x1344), achieving comparable performance with Gemini Pro in understanding scene-text and matches GPT-4V in low hallucination rates. @@ -29,7 +28,9 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> ## News <!-- omit in toc --> #### 📌 Pinned -* [2024.07.19] MiniCPM-Llama3-V 2.5 supports vLLM now! See [here](#vllm). +* [2024.08.06] 🔥🔥🔥 We open-source MiniCPM-V 2.6, which outperforms GPT-4V on single image, multi-image and video understanding. It advances popular features of MiniCPM-Llama3-V 2.5, and can support real-time video understanding on iPad. Try it now! +* [2024.08.03] MiniCPM-Llama3-V 2.5 technical report is released! See [here](https://arxiv.org/abs/2408.01800). +* [2024.07.19] MiniCPM-Llama3-V 2.5 supports vLLM now! See [here](#inference-with-vllm). * [2024.05.28] 🚀🚀🚀 MiniCPM-Llama3-V 2.5 now fully supports its feature in llama.cpp and ollama! Please pull the latest code **of our provided forks** ([llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md), [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5)). GGUF models in various sizes are available [here](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf/tree/main). MiniCPM-Llama3-V 2.5 series is **not supported by the official repositories yet**, and we are working hard to merge PRs. Please stay tuned! * [2024.05.28] 💫 We now support LoRA fine-tuning for MiniCPM-Llama3-V 2.5, using only 2 V100 GPUs! See more statistics [here](https://github.com/OpenBMB/MiniCPM-V/tree/main/finetune#model-fine-tuning-memory-usage-statistics). * [2024.05.23] 🔍 We've released a comprehensive comparison between Phi-3-vision-128k-instruct and MiniCPM-Llama3-V 2.5, including benchmarks evaluations, multilingual capabilities, and inference efficiency 🌟📊🌍🚀. Click [here](./docs/compare_with_phi-3_vision.md) to view more details. @@ -37,11 +38,14 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> <br> +<details> +<summary>Click to view more news.</summary> + * [2024.06.03] Now, you can run MiniCPM-Llama3-V 2.5 on multiple low VRAM GPUs(12 GB or 16 GB) by distributing the model's layers across multiple GPUs. For more details, Check this [link](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md). * [2024.05.25] MiniCPM-Llama3-V 2.5 now supports streaming outputs and customized system prompts. Try it [here](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5#usage)! * [2024.05.24] We release the MiniCPM-Llama3-V 2.5 [gguf](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf), which supports [llama.cpp](#inference-with-llamacpp) inference and provides a 6~8 token/s smooth decoding on mobile phones. Try it now! * [2024.05.20] We open-soure MiniCPM-Llama3-V 2.5, it has improved OCR capability and supports 30+ languages, representing the first end-side MLLM achieving GPT-4V level performance! We provide [efficient inference](#deployment-on-mobile-phone) and [simple fine-tuning](./finetune/readme.md). Try it now! -* [2024.04.23] MiniCPM-V-2.0 supports vLLM now! Click [here](#vllm) to view more details. +* [2024.04.23] MiniCPM-V-2.0 supports vLLM now! Click [here](#inference-with-vllm) to view more details. * [2024.04.18] We create a HuggingFace Space to host the demo of MiniCPM-V 2.0 at [here](https://huggingface.co/spaces/openbmb/MiniCPM-V-2)! * [2024.04.17] MiniCPM-V-2.0 supports deploying [WebUI Demo](#webui-demo) now! * [2024.04.15] MiniCPM-V-2.0 now also supports [fine-tuning](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v-2最佳实践.md) with the SWIFT framework! @@ -49,29 +53,803 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> * [2024.03.14] MiniCPM-V now supports [fine-tuning](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v最佳实践.md) with the SWIFT framework. Thanks to [Jintao](https://github.com/Jintao-Huang) for the contribution！ * [2024.03.01] MiniCPM-V now can be deployed on Mac! * [2024.02.01] We open-source MiniCPM-V and OmniLMM-12B, which support efficient end-side deployment and powerful multimodal capabilities correspondingly. +</details> ## Contents <!-- omit in toc --> +- [MiniCPM-V 2.6](#minicpm-v-26) - [MiniCPM-Llama3-V 2.5](#minicpm-llama3-v-25) - [MiniCPM-V 2.0](#minicpm-v-20) -- [Chat with Our Demo on Gradio](#chat-with-our-demo-on-gradio) +- [Chat with Our Demo on Gradio 🤗](#chat-with-our-demo-on-gradio-) - [Install](#install) - [Inference](#inference) - [Model Zoo](#model-zoo) - [Multi-turn Conversation](#multi-turn-conversation) + - [Chat with multiple images](#chat-with-multiple-images) + - [In-context few-shot learning](#in-context-few-shot-learning) + - [Chat with video](#chat-with-video) + - [Inference on Multiple GPUs](#inference-on-multiple-gpus) - [Inference on Mac](#inference-on-mac) - [Deployment on Mobile Phone](#deployment-on-mobile-phone) - [Inference with llama.cpp](#inference-with-llamacpp) + - [Inference with ollama](#inference-with-ollama) - [Inference with vLLM](#inference-with-vllm) - [Fine-tuning](#fine-tuning) -- [TODO](#todo) -- [🌟 Star History](#-star-history) -- [Citation](#citation) +- [FAQs](#faqs) + + +## MiniCPM-V 2.6 + +**MiniCPM-V 2.6** is the latest and most capable model in the MiniCPM-V series. The model is built on SigLip-400M and Qwen2-7B with a total of 8B parameters. It exhibits a significant performance improvement over MiniCPM-Llama3-V 2.5, and introduces new features for multi-image and video understanding. Notable features of MiniCPM-V 2.6 include: + +- 🔥 **Leading Performance.** + MiniCPM-V 2.6 achieves an average score of 65.2 on the latest version of OpenCompass, a comprehensive evaluation over 8 popular benchmarks. **With only 8B parameters, it surpasses widely used proprietary models like GPT-4o mini, GPT-4V, Gemini 1.5 Pro, and Claude 3.5 Sonnet** for single image understanding. + +- 🖼️ **Multi Image Understanding and In-context Learning.** MiniCPM-V 2.6 can also perform **conversation and reasoning over multiple images**. It achieves **state-of-the-art performance** on popular multi-image benchmarks such as Mantis-Eval, BLINK, Mathverse mv and Sciverse mv, and also shows promising in-context learning capability. + +- 🎬 **Video Understanding.** MiniCPM-V 2.6 can also **accept video inputs**, performing conversation and providing dense captions for spatial-temporal information. It outperforms **GPT-4V, Claude 3.5 Sonnet and LLaVA-NeXT-Video-34B** on Video-MME with/without subtitles. + +- 💪 **Strong OCR Capability and Others.** + MiniCPM-V 2.6 can process images with any aspect ratio and up to 1.8 million pixels (e.g., 1344x1344). It achieves **state-of-the-art performance on OCRBench, surpassing proprietary models such as GPT-4o, GPT-4V, and Gemini 1.5 Pro**. + Based on the the latest [RLAIF-V](https://github.com/RLHF-V/RLAIF-V/) and [VisCPM](https://github.com/OpenBMB/VisCPM) techniques, it features **trustworthy behaviors**, with significantly lower hallucination rates than GPT-4o and GPT-4V on Object HalBench, and supports **multilingual capabilities** on English, Chinese, German, French, Italian, Korean, etc. + + +- 🚀 **Superior Efficiency.** + In addition to its friendly size, MiniCPM-V 2.6 also shows **state-of-the-art token density** (i.e., number of pixels encoded into each visual token). **It produces only 640 tokens when processing a 1.8M pixel image, which is 75% fewer than most models**. This directly improves the inference speed, first-token latency, memory usage, and power consumption. As a result, MiniCPM-V 2.6 can efficiently support **real-time video understanding** on end-side devices such as iPad. + +- 💫 **Easy Usage.** +MiniCPM-V 2.6 can be easily used in various ways: (1) [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpmv-main/examples/llava/README-minicpmv2.6.md) and [ollama](https://github.com/OpenBMB/ollama/blob/minicpm-v2.6/examples/minicpm-v2.6/README.md) support for efficient CPU inference on local devices, (2) [int4](https://huggingface.co/openbmb/MiniCPM-V-2_6-int4) and [GGUF](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) format quantized models in 16 sizes, (3) [vLLM](#inference-with-vllm) support for high-throughput and memory-efficient inference, (4) fine-tuning on new domains and tasks, (5) quick local WebUI demo setup with [Gradio](#chat-with-our-demo-on-gradio), and (6) online web [demo](https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6). + +### Evaluation <!-- omit in toc --> +<div align="center"> + <img src=assets/radar_final.png width=66% /> +</div> + +<details> +<summary>Click to view single image results on OpenCompass, MME, MMVet, OCRBench, MMMU, MathVista, MMB, AI2D, TextVQA, DocVQA, HallusionBench, Object HalBench. </summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Token Density<sup>+</sup></th> + <th>OpenCompass</th> + <th>MME</th> + <th>MMVet</th> + <th>OCRBench</th> + <th>MMMU val</th> + <th>MathVista mini</th> + <th>MMB1.1 test</th> + <th>AI2D</th> + <th>TextVQA val</th> + <th>DocVQA test</th> + <th>HallusionBench</th> + <th>Object HalBench</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="15" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4o</td> + <td>-</td> + <td>1088</td> + <td>69.9</td> + <td>2328.7</td> + <td>69.1</td> + <td>736</td> + <td>69.2</td> + <td>61.3</td> + <td>82.2</td> + <td>84.6</td> + <td>-</td> + <td>92.8</td> + <td>55.0</td> + <td>17.6</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Claude 3.5 Sonnet</td> + <td>-</td> + <td>750</td> + <td>67.9</td> + <td>1920.0</td> + <td>66.0</td> + <td>788</td> + <td>65.9</td> + <td>61.6</td> + <td>78.5</td> + <td>80.2</td> + <td>-</td> + <td>95.2</td> + <td>49.9</td> + <td>13.8</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Gemini 1.5 Pro</td> + <td>-</td> + <td>-</td> + <td>64.4</td> + <td>2110.6</td> + <td>64.0</td> + <td>754</td> + <td>60.6</td> + <td>57.7</td> + <td>73.9</td> + <td>79.1</td> + <td>73.5</td> + <td>86.5</td> + <td>45.6</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4o mini</td> + <td>-</td> + <td>1088</td> + <td>64.1</td> + <td>2003.4</td> + <td>66.9</td> + <td>785</td> + <td>60.0</td> + <td>52.4</td> + <td>76.0</td> + <td>77.8</td> + <td>-</td> + <td>-</td> + <td>46.1</td> + <td>12.4</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>1088</td> + <td>63.5</td> + <td>2070.2</td> + <td>67.5</td> + <td>656</td> + <td>61.7</td> + <td>54.7</td> + <td>79.8</td> + <td>78.6</td> + <td>78.0</td> + <td>87.2</td> + <td>43.9</td> + <td>14.2</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Step-1V</td> + <td>-</td> + <td>-</td> + <td>59.5</td> + <td>2206.4</td> + <td>63.3</td> + <td>625</td> + <td>49.9</td> + <td>44.8</td> + <td>78.0</td> + <td>79.2</td> + <td>71.6</td> + <td>-</td> + <td>48.4</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Qwen-VL-Max</td> + <td>-</td> + <td>784</td> + <td>58.3</td> + <td>2281.7</td> + <td>61.8</td> + <td>684</td> + <td>52.0</td> + <td>43.4</td> + <td>74.6</td> + <td>75.7</td> + <td>79.5</td> + <td>93.1</td> + <td>41.2</td> + <td>13.4</td> + </tr> + <tr> + <td colspan="15" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Yi-34B</td> + <td>34B</td> + <td>157</td> + <td>55.0</td> + <td>2006.5</td> + <td>50.7</td> + <td>574</td> + <td>48.8</td> + <td>40.4</td> + <td>77.8</td> + <td>78.9</td> + <td>69.3</td> + <td>-</td> + <td>34.8</td> + <td>12.6</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Mini-Gemini-HD-34B</td> + <td>34B</td> + <td>157</td> + <td>-</td> + <td>2141.0</td> + <td>59.3</td> + <td>518</td> + <td>48.0</td> + <td>43.3</td> + <td>-</td> + <td>80.5</td> + <td>74.1</td> + <td>78.9</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Cambrian-34B</td> + <td>34B</td> + <td>1820</td> + <td>58.3</td> + <td>2049.9</td> + <td>53.2</td> + <td>591</td> + <td>50.4</td> + <td>50.3</td> + <td>77.8</td> + <td>79.5</td> + <td>76.7</td> + <td>75.5</td> + <td>41.6</td> + <td>14.7</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GLM-4V-9B</td> + <td>13B</td> + <td>784</td> + <td>59.1</td> + <td>2018.8</td> + <td>58.0</td> + <td>776</td> + <td>46.9</td> + <td>51.1</td> + <td>67.9</td> + <td>71.2</td> + <td>-</td> + <td>-</td> + <td>45.0</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>706</td> + <td>64.1</td> + <td>2215.1</td> + <td>54.3</td> + <td>794</td> + <td><strong>51.2</strong></td> + <td>58.3</td> + <td><strong>79.4</strong></td> + <td><strong>83.6</strong></td> + <td>77.4</td> + <td><strong>91.6</strong></td> + <td>45.0</td> + <td>21.3</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">MiniCPM-Llama-V 2.5</td> + <td>8B</td> + <td>1882</td> + <td>58.8</td> + <td>2024.6</td> + <td>52.8</td> + <td>725</td> + <td>45.8</td> + <td>54.3</td> + <td>72.0</td> + <td>78.4</td> + <td>76.6</td> + <td>84.8</td> + <td>42.4</td> + <td>10.3</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>2822</strong></td> + <td><strong>65.2</strong></td> + <td><strong>2348.4</strong>*</td> + <td><strong>60.0</strong></td> + <td><strong>852</strong>*</td> + <td>49.8*</td> + <td><strong>60.6</strong></td> + <td>78.0</td> + <td>82.1</td> + <td><strong>80.1<strong></td> + <td>90.8</td> + <td><strong>48.1</strong>*</td> + <td><strong>8.2</strong></td> + </tr> + </tbody> +</table> + +</div> +* We evaluate this benchmark using chain-of-thought prompting. Specifically, for MME, we used this technique only for the Cognition set. + +<sup>+</sup> Token Density: number of pixels encoded into each visual token at maximum resolution, i.e., # pixels at maximum resolution / # visual tokens. + +Note: For proprietary models, we calculate token density based on the image encoding charging strategy defined in the official API documentation, which provides an upper-bound estimation. + +</details> + + +<details> +<summary>Click to view multi-image results on Mantis Eval, BLINK, Mathverse mv, Sciverse mv, MIRB.</summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Mantis Eval</th> + <th>BLINK val</th> + <th>Mathverse mv</th> + <th>Sciverse mv</th> + <th>MIRB</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="7" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>62.7</td> + <td>54.6</td> + <td>60.3</td> + <td>66.9</td> + <td>53.1</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Interleave-14B</td> + <td>14B</td> + <td>66.4</td> + <td>52.6</td> + <td>32.7</td> + <td>30.2</td> + <td>-</td> + </tr> + <tr> + <td colspan="7" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Emu2-Chat</td> + <td>37B</td> + <td>37.8</td> + <td>36.2</td> + <td>-</td> + <td>27.2</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">CogVLM</td> + <td>17B</td> + <td>45.2</td> + <td>41.1</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">VPG-C</td> + <td>7B</td> + <td>52.4</td> + <td>43.1</td> + <td>24.3</td> + <td>23.1</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">VILA 8B</td> + <td>8B</td> + <td>51.2</td> + <td>39.3</td> + <td>-</td> + <td>36.5</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternLM-XComposer-2.5</td> + <td>8B</td> + <td>53.1*</td> + <td>48.9</td> + <td>32.1*</td> + <td>-</td> + <td>42.5</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>59.0*</td> + <td>50.9</td> + <td>30.5*</td> + <td>34.4*</td> + <td><strong>56.9*</strong></td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>69.1</strong></td> + <td><strong>53.0</strong></td> + <td><strong>84.9</strong></td> + <td><strong>74.9</strong></td> + <td>53.8</td> + </tr> + </tbody> +</table> + +</div> +* We evaluate the officially released checkpoint by ourselves. +</details> + +<details> +<summary>Click to view video results on Video-MME and Video-ChatGPT.</summary> +<div align="center"> +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th colspan="2">Video-MME</th> + <th colspan="5">Video-ChatGPT</th> + </tr> + <tr> + <th align="left"></th> + <th></th> + <th>w/o subs</th> + <th>w subs</th> + <th>Correctness</th> + <th>Detail</th> + <th>Context</th> + <th>Temporal</th> + <th>Consistency</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="9" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Claude 3.5 Sonnet</td> + <td>-</td> + <td>60.0</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>59.9</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td colspan="9" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-7B</td> + <td>7B</td> + <td>-</td> + <td>-</td> + <td>3.39</td> + <td>3.29</td> + <td>3.92</td> + <td>2.60</td> + <td>3.12</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-34B</td> + <td>34B</td> + <td>-</td> + <td>-</td> + <td>3.29</td> + <td>3.23</td> + <td>3.83</td> + <td>2.51</td> + <td>3.47</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">CogVLM2-Video</td> + <td>12B</td> + <td>-</td> + <td>-</td> + <td>3.49</td> + <td><strong>3.46</strong></td> + <td>3.23</td> + <td><strong>2.98</strong></td> + <td><strong>3.64</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LongVA</td> + <td>7B</td> + <td>52.4</td> + <td>54.3</td> + <td>3.05</td> + <td>3.09</td> + <td>3.77</td> + <td>2.44</td> + <td><strong>3.64</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>54.0</td> + <td>56.9</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternLM-XComposer-2.5</td> + <td>8B</td> + <td>55.8</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Video</td> + <td>32B</td> + <td>60.2</td> + <td>63.0</td> + <td>3.48</td> + <td>3.37</td> + <td><strong>3.95</strong></td> + <td>2.64</td> + <td>3.28</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>60.9</strong></td> + <td><strong>63.6</strong></td> + <td><strong>3.59</strong></td> + <td>3.28</td> + <td>3.93</td> + <td>2.73</td> + <td>3.62</td> + </tr> + </tbody> +</table> +</div> +</details> + + +<details> +<summary>Click to view few-shot results on TextVQA, VizWiz, VQAv2, OK-VQA.</summary> +<div align="center"> +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Shot</th> + <th>TextVQA val</th> + <th>VizWiz test-dev</th> + <th>VQAv2 test-dev</th> + <th>OK-VQA val</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">Flamingo</td> + <td rowspan="3">80B</td> + <td>0*</td> + <td>35.0</td> + <td>31.6</td> + <td>56.3</td> + <td>40.6</td> + </tr> + <tr> + <td>4</td> + <td>36.5</td> + <td>39.6</td> + <td>63.1</td> + <td><strong>57.4</strong></td> + </tr> + <tr> + <td>8</td> + <td>37.3</td> + <td>44.8</td> + <td>65.6</td> + <td>57.5</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">IDEFICS</td> + <td rowspan="3">80B</td> + <td>0*</td> + <td>30.9</td> + <td>36.0</td> + <td>60.0</td> + <td>45.2</td> + </tr> + <tr> + <td>4</td> + <td>34.3</td> + <td>40.4</td> + <td>63.6</td> + <td>52.4</td> + </tr> + <tr> + <td>8</td> + <td>35.7</td> + <td>46.1</td> + <td>64.8</td> + <td>55.1</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">OmniCorpus</td> + <td rowspan="3">7B</td> + <td>0*</td> + <td>43.0</td> + <td>49.8</td> + <td>63.2</td> + <td>45.5</td> + </tr> + <tr> + <td>4</td> + <td>45.4</td> + <td>51.3</td> + <td>64.5</td> + <td>46.5</td> + </tr> + <tr> + <td>8</td> + <td>45.6</td> + <td>52.2</td> + <td>64.7</td> + <td>46.6</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">Emu2</td> + <td rowspan="3">37B</td> + <td>0</td> + <td>26.4</td> + <td>40.4</td> + <td>33.5</td> + <td>26.7</td> + </tr> + <tr> + <td>4</td> + <td>48.2</td> + <td>54.6</td> + <td>67.0</td> + <td>53.2</td> + </tr> + <tr> + <td>8</td> + <td>49.3</td> + <td>54.7</td> + <td>67.8</td> + <td>54.1</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="2">MM1</td> + <td rowspan="2">30B</td> + <td>0</td> + <td>26.2</td> + <td>40.4</td> + <td>48.9</td> + <td>26.7</td> + </tr> + <tr> + <td>8</td> + <td>49.3</td> + <td>54.7</td> + <td><strong>70.9</strong></td> + <td>54.1</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td align="left" nowrap="nowrap" rowspan="3">MiniCPM-V 2.6<sup>+</sup></td> + <td rowspan="3">8B</td> + <td>0</td> + <td>43.9</td> + <td>33.8</td> + <td>45.4</td> + <td>23.9</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td>4</td> + <td>63.6</td> + <td>60.5</td> + <td>65.5</td> + <td>50.1</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td>8</td> + <td><strong>64.6</strong></td> + <td><strong>63.4</strong></td> + <td>68.2</td> + <td>51.4</td> + </tr> + </tbody> +</table> + + +</div> +* denotes zero image shot and two additional text shots following Flamingo. + +<sup>+</sup> We evaluate the pretraining ckpt without SFT. +</details> + +### Examples <!-- omit in toc --> + +<div style="display: flex; flex-direction: column; align-items: center;"> + <img src="assets/minicpmv2_6/multi_img-bike.png" alt="Bike" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multi_img-menu.png" alt="Menu" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multi_img-code.png" alt="Code" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/ICL-Mem.png" alt="Mem" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multiling-medal.png" alt="medal" style="margin-bottom: 10px;"> +</div> +<details> + <summary>Click to view more cases.</summary> + <div style="display: flex; flex-direction: column; align-items: center;"> + <img src="assets/minicpmv2_6/ICL-elec.png" alt="elec" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multiling-olympic.png" alt="Menu" style="margin-bottom: 10px;"> + </div> +</details> + +We deploy MiniCPM-V 2.6 on end devices. The demo video is the raw screen recording on a iPad Pro without edition. + +<table align="center"> + <p align="center"> + <img src="assets/gif_cases/ai.gif" width=32%/> +      + <img src="assets/gif_cases/beer.gif" width=32%/> + </p> +</table> + +<table align="center"> + <p align="center"> + <img src="assets/gif_cases/ticket.gif" width=32%/> +      + <img src="assets/gif_cases/wfh.gif" width=32%/> + </p> +</table> + +<table align="center"> + <p align="center"> + <video src="https://github.com/user-attachments/assets/21f4b818-ede1-4822-920e-91281725c830" width="360" /> </video> + <!-- <video src="https://github.com/user-attachments/assets/c835f757-206b-4d9c-8e36-70d67b453628" width="360" /> </video> --> + </p> +</table> ## MiniCPM-Llama3-V 2.5 +<details> +<summary>Click to view more details of MiniCPM-Llama3-V 2.5</summary> + **MiniCPM-Llama3-V 2.5** is the latest model in the MiniCPM-V series. The model is built on SigLip-400M and Llama3-8B-Instruct with a total of 8B parameters. It exhibits a significant performance improvement over MiniCPM-V 2.0. Notable features of MiniCPM-Llama3-V 2.5 include: - 🔥 **Leading Performance.** @@ -92,15 +870,6 @@ Join our <a href="docs/wechat.md" target="_blank"> 💬 WeChat</a> - 💫 **Easy Usage.** MiniCPM-Llama3-V 2.5 can be easily used in various ways: (1) [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) and [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5) support for efficient CPU inference on local devices, (2) [GGUF](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf) format quantized models in 16 sizes, (3) efficient [LoRA](https://github.com/OpenBMB/MiniCPM-V/tree/main/finetune#lora-finetuning) fine-tuning with only 2 V100 GPUs, (4) [streaming output](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5#usage), (5) quick local WebUI demo setup with [Gradio](https://github.com/OpenBMB/MiniCPM-V/blob/main/web_demo_2.5.py) and [Streamlit](https://github.com/OpenBMB/MiniCPM-V/blob/main/web_demo_streamlit-2_5.py), and (6) interactive demos on [HuggingFace Spaces](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5). -### MiniCPM-Llama3-V 2.5 Common Module Navigation <!-- omit in toc --> -You can click links in the following table to quickly access the commonly used functions for MiniCPM-Llama3-V 2.5. -| Functional Categories | | | | | | | || -|:--------:|:------:|:--------------:|:--------:|:-------:|:-----------:|:-----------:|:--------:|:-----------:| -| Inference | [Transformers](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md) | [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5) | [SWIFT](./docs/swift_train_and_infer.md) | [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) | [Xinfrence](./docs/xinference_infer.md) | [Gradio](./web_demo_2.5.py) | [Streamlit](./web_demo_streamlit-2_5.py) |[vLLM](#vllm) -| Finetune | [Full-parameter](./finetune/readme.md) | [Lora](./finetune/readme.md) | [SWIFT](./docs/swift_train_and_infer.md) | | | | | | -| Edge Deployment | [apk](http://minicpm.modelbest.cn/android/modelbest-release-20240528_182155.apk) | [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) | | | | | | | -| Quantize | [Bnb](./quantize/bnb_quantize.py) | - ### Evaluation <!-- omit in toc --> <div align="center"> @@ -399,20 +1168,8 @@ You can click links in the following table to quickly access the commonly used f </p> </table> -We deploy MiniCPM-Llama3-V 2.5 on end devices. The demo video is the raw screen recording on a Xiaomi 14 Pro without edition. +</details> -<table align="center"> - <p align="center"> - <img src="assets/gif_cases/ticket.gif" width=32%/> - <img src="assets/gif_cases/meal_plan.gif" width=32%/> - </p> -</table> - -<table align="center"> - <p align="center"> - <img src="assets/gif_cases/1-4.gif" width=64%/> - </p> -</table> ## MiniCPM-V 2.0 @@ -469,14 +1226,14 @@ We deploy MiniCPM-V 2.0 on end devices. The demo video is the raw screen recordi | OmniLMM-12B | [Document](./omnilmm_en.md) | -## Chat with Our Demo on Gradio +## Chat with Our Demo on Gradio 🤗 -We provide online and local demos powered by HuggingFace [Gradio](https://github.com/gradio-app/gradio), the most popular model deployment framework nowadays. It supports streaming outputs, progress bars, queuing, alerts, and other useful features. +We provide online and local demos powered by Hugging Face Gradio <a href='https://github.com/gradio-app/gradio'><img src='https://img.shields.io/github/stars/gradio-app/gradio'></a>, the most popular model deployment framework nowadays. It supports streaming outputs, progress bars, queuing, alerts, and other useful features. ### Online Demo <!-- omit in toc --> -Click here to try out the online demo of [MiniCPM-Llama3-V 2.5](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) ｜ [MiniCPM-V 2.0](https://huggingface.co/spaces/openbmb/MiniCPM-V-2) on HuggingFace Spaces. +Click here to try out the online demo of [MiniCPM-V 2.6](https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6) | [MiniCPM-Llama3-V 2.5](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) | [MiniCPM-V 2.0](https://huggingface.co/spaces/openbmb/MiniCPM-V-2). ### Local WebUI Demo <!-- omit in toc --> @@ -488,10 +1245,8 @@ pip install -r requirements.txt ```shell # For NVIDIA GPUs, run: -python web_demo_2.5.py --device cuda +python web_demo_2.6.py --device cuda -# For Mac with MPS (Apple silicon or AMD GPUs), run: -PYTORCH_ENABLE_MPS_FALLBACK=1 python web_demo_2.5.py --device mps ``` @@ -524,9 +1279,12 @@ pip install -r requirements.txt | Model | Device | Memory |          Description | Download | |:-----------|:--:|:-----------:|:-------------------|:---------------:| -| MiniCPM-Llama3-V 2.5 | GPU | 19 GB | The lastest version, achieving state-of-the end-side multimodal performance. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5) | -| MiniCPM-Llama3-V 2.5 gguf | CPU | 5 GB | The gguf version, lower memory usage and faster inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)   [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-gguf) | -| MiniCPM-Llama3-V 2.5 int4 | GPU | 8 GB | The int4 quantized version，lower GPU memory usage. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-int4/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-int4) | +| MiniCPM-V 2.6| GPU | 17 GB | The latest version, achieving state-of-the-art end-side performance for single image, multi-image and video understanding. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6) | +| MiniCPM-V 2.6 gguf | CPU | 6 GB | The gguf version, lower memory usage and faster inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6-gguf) | +| MiniCPM-V 2.6 int4 | GPU | 7 GB | The int4 quantized version, lower GPU memory usage. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6-int4)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6-int4) | +| MiniCPM-Llama3-V 2.5 | GPU | 19 GB | Strong end-side multimodal performance. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5) | +| MiniCPM-Llama3-V 2.5 gguf | CPU | 6 GB | The gguf version, lower memory usage and faster inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)   [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-gguf) | +| MiniCPM-Llama3-V 2.5 int4 | GPU | 8 GB | The int4 quantized version, lower GPU memory usage. | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-int4/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-int4) | | MiniCPM-V 2.0 | GPU | 8 GB | Light version, balance the performance the computation cost. | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2) | | MiniCPM-V 1.0 | GPU | 7 GB | Lightest version, achieving the fastest inference. | [🤗](https://huggingface.co/openbmb/MiniCPM-V)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V) | @@ -540,30 +1298,40 @@ Please refer to the following codes to run. ```python -from chat import MiniCPMVChat, img2base64 import torch -import json +from PIL import Image +from transformers import AutoModel, AutoTokenizer torch.manual_seed(0) -chat_model = MiniCPMVChat('openbmb/MiniCPM-Llama3-V-2_5') +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) -im_64 = img2base64('./assets/airplane.jpeg') +image = Image.open('./assets/airplane.jpeg').convert('RGB') # First round chat -msgs = [{"role": "user", "content": "Tell me the model of this aircraft."}] +question = "Tell me the model of this aircraft." +msgs = [{'role': 'user', 'content': [image, question]}] -inputs = {"image": im_64, "question": json.dumps(msgs)} -answer = chat_model.chat(inputs) +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) print(answer) # Second round chat # pass history context of multi-turn conversation -msgs.append({"role": "assistant", "content": answer}) -msgs.append({"role": "user", "content": "Introduce something about Airbus A380."}) +msgs.append({"role": "assistant", "content": [answer]}) +msgs.append({"role": "user", "content": ["Introduce something about Airbus A380."]}) -inputs = {"image": im_64, "question": json.dumps(msgs)} -answer = chat_model.chat(inputs) +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) print(answer) ``` @@ -575,6 +1343,127 @@ You will get the following output: "The Airbus A380 is a double-deck, wide-body, four-engine jet airliner made by Airbus. It is the world's largest passenger airliner and is known for its long-haul capabilities. The aircraft was developed to improve efficiency and comfort for passengers traveling over long distances. It has two full-length passenger decks, which can accommodate more passengers than a typical single-aisle airplane. The A380 has been operated by airlines such as Lufthansa, Singapore Airlines, and Emirates, among others. It is widely recognized for its unique design and significant impact on the aviation industry." ``` +#### Chat with multiple images +<details> +<summary> Click to view Python code running MiniCPM-V 2.6 with multiple images input. </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +image1 = Image.open('image1.jpg').convert('RGB') +image2 = Image.open('image2.jpg').convert('RGB') +question = 'Compare image 1 and image 2, tell me about the differences between image 1 and image 2.' + +msgs = [{'role': 'user', 'content': [image1, image2, question]}] + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) +print(answer) +``` +</details> + +#### In-context few-shot learning +<details> +<summary> Click to view Python code running MiniCPM-V 2.6 with few-shot input. </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +question = "production date" +image1 = Image.open('example1.jpg').convert('RGB') +answer1 = "2023.08.04" +image2 = Image.open('example2.jpg').convert('RGB') +answer2 = "2007.04.24" +image_test = Image.open('test.jpg').convert('RGB') + +msgs = [ + {'role': 'user', 'content': [image1, question]}, {'role': 'assistant', 'content': [answer1]}, + {'role': 'user', 'content': [image2, question]}, {'role': 'assistant', 'content': [answer2]}, + {'role': 'user', 'content': [image_test, question]} +] + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) +print(answer) +``` +</details> + +#### Chat with video +<details> +<summary> Click to view Python code running MiniCPM-V 2.6 with video input. </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer +from decord import VideoReader, cpu # pip install decord + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +MAX_NUM_FRAMES=64 # if cuda OOM set a smaller number + +def encode_video(video_path): + def uniform_sample(l, n): + gap = len(l) / n + idxs = [int(i * gap + gap / 2) for i in range(n)] + return [l[i] for i in idxs] + + vr = VideoReader(video_path, ctx=cpu(0)) + sample_fps = round(vr.get_avg_fps() / 1) # FPS + frame_idx = [i for i in range(0, len(vr), sample_fps)] + if len(frame_idx) > MAX_NUM_FRAMES: + frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES) + frames = vr.get_batch(frame_idx).asnumpy() + frames = [Image.fromarray(v.astype('uint8')) for v in frames] + print('num frames:', len(frames)) + return frames + +video_path="video_test.mp4" +frames = encode_video(video_path) +question = "Describe the video" +msgs = [ + {'role': 'user', 'content': frames + [question]}, +] + +# Set decode params for video +params = {} +params["use_image_id"] = False +params["max_slice_nums"] = 2 # use 1 if cuda OOM and video resolution > 448*448 + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + **params +) +print(answer) +``` +</details> + + ### Inference on Multiple GPUs You can run MiniCPM-Llama3-V 2.5 on multiple low VRAM GPUs (12 GB or 16 GB) by distributing the model's layers across multiple GPUs. Please refer to this [tutorial](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md) for detailed instructions on how to load the model and inference using multiple low VRAM GPUs. @@ -615,33 +1504,100 @@ PYTORCH_ENABLE_MPS_FALLBACK=1 python test.py </details> ### Deployment on Mobile Phone -MiniCPM-Llama3-V 2.5 and MiniCPM-V 2.0 can be deployed on mobile phones with Android operating systems. 🚀 Click [MiniCPM-Llama3-V 2.5](http://minicpm.modelbest.cn/android/modelbest-release-20240528_182155.apk) / [MiniCPM-V 2.0](https://github.com/OpenBMB/mlc-MiniCPM) to install apk. +MiniCPM-V 2.0 can be deployed on mobile phones with Android operating systems. 🚀 Click [MiniCPM-V 2.0](https://github.com/OpenBMB/mlc-MiniCPM) to install apk. -### Inference with llama.cpp<a id="inference-with-llamacpp"></a> -MiniCPM-Llama3-V 2.5 can run with llama.cpp now! See our fork of [llama.cpp](https://github.com/OpenBMB/llama.cpp/tree/minicpm-v2.5/examples/minicpmv) for more detail. This implementation supports smooth inference of 6~8 token/s on mobile phones (test environment：Xiaomi 14 pro + Snapdragon 8 Gen 3). +### Inference with llama.cpp +MiniCPM-V 2.6 can run with llama.cpp now! See [our fork of llama.cpp](https://github.com/OpenBMB/llama.cpp/tree/minicpmv-main/examples/llava/README-minicpmv2.6.md) for more detail. This implementation supports smooth inference of 16~18 token/s on iPad (test environment：iPad Pro + M4). -### Inference with vLLM<a id="vllm"></a> +### Inference with ollama +MiniCPM-V 2.6 can run with ollama now! See [our fork of ollama](https://github.com/OpenBMB/ollama/blob/minicpm-v2.6/examples/minicpm-v2.6/README.md) for more detail. This implementation supports smooth inference of 16~18 token/s on iPad (test environment：iPad Pro + M4). + +### Inference with vLLM <details> -<summary> vLLM now officially supports MiniCPM-V 2.0 and MiniCPM-Llama3-V 2.5, Click to see. </summary> +<summary> vLLM now officially supports MiniCPM-V 2.6, MiniCPM-Llama3-V 2.5 and MiniCPM-V 2.0, Click to see. </summary> -1. Clone the official vLLM: +1. Install vLLM(>=0.5.4): ```shell -git clone https://github.com/vllm-project/vllm.git +pip install vllm ``` -2. Install vLLM: -```shell -cd vllm -pip install -e . -``` -3. Install timm: +2. Install timm: (optional, MiniCPM-V 2.0 need timm) ```shell pip install timm==0.9.10 ``` -4. Run the example:（If you use model in local path, please update the model code to the latest version on Hugging Face.) -```shell -python examples/minicpmv_example.py +3. Run the example(for image): +```python +from transformers import AutoTokenizer +from PIL import Image +from vllm import LLM, SamplingParams + +MODEL_NAME = "openbmb/MiniCPM-V-2_6" +# Also available for previous models +# MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5" +# MODEL_NAME = "HwwwH/MiniCPM-V-2" + +image = Image.open("xxx.png").convert("RGB") +tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) +llm = LLM( + model=MODEL_NAME, + trust_remote_code=True, + gpu_memory_utilization=1, + max_model_len=2048 +) + +messages = [{ + "role": + "user", + "content": + # Number of images + "(<image>./</image>)" + \ + "\nWhat is the content of this image?" +}] +prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True +) + +# Single Inference +inputs = { + "prompt": prompt, + "multi_modal_data": { + "image": image + # Multi images, the number of images should be equal to that of `(<image>./</image>)` + # "image": [image, image] + }, +} +# Batch Inference +# inputs = [{ +# "prompt": prompt, +# "multi_modal_data": { +# "image": image +# }, +# } for _ in 2] + + +# 2.6 +stop_tokens = ['<|im_end|>', '<|endoftext|>'] +stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] +# 2.0 +# stop_token_ids = [tokenizer.eos_id] +# 2.5 +# stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id] + +sampling_params = SamplingParams( + stop_token_ids=stop_token_ids, + use_beam_search=True, + temperature=0, + best_of=3, + max_tokens=1024 +) + +outputs = llm.generate(inputs, sampling_params=sampling_params) + +print(outputs[0].outputs[0].text) ``` +4. click [here](https://modelbest.feishu.cn/wiki/C2BWw4ZP0iCDy7kkCPCcX2BHnOf?from=from_copylink) if you want to use it with *video*, or get more details about `vLLM`. </details> ## Fine-tuning @@ -658,12 +1614,8 @@ We now support MiniCPM-V series fine-tuning with the SWIFT framework. SWIFT supp Best Practices：[MiniCPM-V 1.0](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v最佳实践.md), [MiniCPM-V 2.0](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v-2最佳实践.md) - - -## TODO - -- [x] MiniCPM-V fine-tuning support -- [ ] Code release for real-time interactive assistant +## FAQs +Click here to view the [FAQs](./docs/faqs.md) ## Model License <!-- omit in toc --> @@ -678,7 +1630,7 @@ Best Practices：[MiniCPM-V 1.0](https://github.com/modelscope/swift/blob/main/d As LMMs, MiniCPM-V models (including OmniLMM) generate contents by learning a large amount of multimodal corpora, but they cannot comprehend, express personal opinions or make value judgement. Anything generated by MiniCPM-V models does not represent the views and positions of the model developers -We will not be liable for any problems arising from the use of MiniCPMV-V models, including but not limited to data security issues, risk of public opinion, or any risks and problems arising from the misdirection, misuse, dissemination or misuse of the model. +We will not be liable for any problems arising from the use of MiniCPM-V models, including but not limited to data security issues, risk of public opinion, or any risks and problems arising from the misdirection, misuse, dissemination or misuse of the model. ## Institutions <!-- omit in toc --> @@ -689,13 +1641,7 @@ This project is developed by the following institutions: - <img src="assets/modelbest.png" width="28px"> [ModelBest](https://modelbest.cn/) - <img src="assets/zhihu.webp" width="28px"> [Zhihu](https://www.zhihu.com/ ) -## Other Multimodal Projects from Our Team <!-- omit in toc --> - -👏 Welcome to explore other multimodal projects of our team: - -[VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) - -## 🌟 Star History +## 🌟 Star History <!-- omit in toc --> <table align="center"> @@ -723,33 +1669,22 @@ This project is developed by the following institutions: /> </picture> --> -## Citation +## Key Techniques and Other Multimodal Projects <!-- omit in toc --> + +👏 Welcome to explore key techniques of MiniCPM-V and other multimodal projects of our team: + +[VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) + + +## Citation <!-- omit in toc --> If you find our model/code/paper helpful, please consider cite our papers 📝 and star us ⭐️！ ```bib -@article{yu2023rlhf, - title={Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback}, - author={Yu, Tianyu and Yao, Yuan and Zhang, Haoye and He, Taiwen and Han, Yifeng and Cui, Ganqu and Hu, Jinyi and Liu, Zhiyuan and Zheng, Hai-Tao and Sun, Maosong and others}, - journal={arXiv preprint arXiv:2312.00849}, - year={2023} -} -@article{viscpm, - title={Large Multilingual Models Pivot Zero-Shot Multimodal Learning across Languages}, - author={Jinyi Hu and Yuan Yao and Chongyi Wang and Shan Wang and Yinxu Pan and Qianyu Chen and Tianyu Yu and Hanghao Wu and Yue Zhao and Haoye Zhang and Xu Han and Yankai Lin and Jiao Xue and Dahai Li and Zhiyuan Liu and Maosong Sun}, - journal={arXiv preprint arXiv:2308.12038}, - year={2023} -} -@article{xu2024llava-uhd, - title={{LLaVA-UHD}: an LMM Perceiving Any Aspect Ratio and High-Resolution Images}, - author={Xu, Ruyi and Yao, Yuan and Guo, Zonghao and Cui, Junbo and Ni, Zanlin and Ge, Chunjiang and Chua, Tat-Seng and Liu, Zhiyuan and Huang, Gao}, - journal={arXiv preprint arXiv:2403.11703}, - year={2024} -} -@article{yu2024rlaifv, - title={RLAIF-V: Aligning MLLMs through Open-Source AI Feedback for Super GPT-4V Trustworthiness}, - author={Yu, Tianyu and Zhang, Haoye and Yao, Yuan and Dang, Yunkai and Chen, Da and Lu, Xiaoman and Cui, Ganqu and He, Taiwen and Liu, Zhiyuan and Chua, Tat-Seng and Sun, Maosong}, - journal={arXiv preprint arXiv:2405.17220}, +@article{yao2024minicpm, + title={MiniCPM-V: A GPT-4V Level MLLM on Your Phone}, + author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others}, + journal={arXiv preprint arXiv:2408.01800}, year={2024} } ``` diff --git a/README_zh.md b/README_zh.md index 07c86e3..7d2125d 100644 --- a/README_zh.md +++ b/README_zh.md @@ -4,7 +4,7 @@ <img src="./assets/minicpmv.png" width="300em" ></img> -**端侧可用的 GPT-4V 级多模态大模型** +**端侧可用的 GPT-4V 级单图、多图、视频多模态大模型** <strong>中文 | [English](./README_en.md)</strong> @@ -12,17 +12,18 @@ 加入我们的 <a href="docs/wechat.md" target="_blank"> 💬 微信社区</a> <p align="center"> - MiniCPM-Llama3-V 2.5 <a href="https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5">🤖</a> | - MiniCPM-V 2.0 <a href="https://huggingface.co/openbmb/MiniCPM-V-2/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-V-2">🤖</a> | - <a href="https://openbmb.vercel.app/minicpm-v-2">MiniCPM-V 2.0 技术博客</a> + MiniCPM-V 2.6 <a href="https://huggingface.co/openbmb/MiniCPM-V-2_6">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6">🤖</a> | MiniCPM-Llama3-V 2.5 <a href="https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/">🤗</a> <a href="https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5">🤖</a> | + <a href=https://arxiv.org/abs/2408.01800>MiniCPM-Llama3-V 2.5 技术报告</a> </p> + </div> -**MiniCPM-V**是面向图文理解的端侧多模态大模型系列。该系列模型接受图像和文本输入，并提供高质量的文本输出。自2024年2月以来，我们共发布了4个版本模型，旨在实现**领先的性能和高效的部署**，目前该系列最值得关注的模型包括： +**MiniCPM-V**是面向图文理解的端侧多模态大模型系列。该系列模型接受图像和文本输入，并提供高质量的文本输出。自2024年2月以来，我们共发布了5个版本模型，旨在实现**领先的性能和高效的部署**，目前该系列最值得关注的模型包括： -- **MiniCPM-Llama3-V 2.5**：🔥🔥🔥 MiniCPM-V系列的最新、性能最佳模型。总参数量8B，多模态综合性能**超越 GPT-4V-1106、Gemini Pro、Claude 3、Qwen-VL-Max 等商用闭源模型**，OCR 能力及指令跟随能力进一步提升，并**支持超过30种语言**的多模态交互。通过系统使用模型量化、CPU、NPU、编译优化等高效推理技术，MiniCPM-Llama3-V 2.5 可以实现**高效的终端设备部署**。 + +- **MiniCPM-V 2.6**: 🔥🔥🔥 MiniCPM-V系列的最新、性能最佳模型。总参数量 8B，单图、多图和视频理解性能**超越了 GPT-4V**。在单图理解上，它取得了优于 **GPT-4o mini、Gemini 1.5 Pro 和 Claude 3.5 Sonnet**等商用闭源模型的表现，并进一步优化了 MiniCPM-Llama3-V 2.5 的 OCR、可信行为、多语言支持以及端侧部署等诸多特性。基于其领先的视觉 token 密度，MiniCPM-V 2.6 成为了首个支持在 iPad 等端侧设备上进行实时视频理解的多模态大模型。 - **MiniCPM-V 2.0**：MiniCPM-V系列的最轻量级模型。总参数量2B，多模态综合性能超越 Yi-VL 34B、CogVLM-Chat 17B、Qwen-VL-Chat 10B 等更大参数规模的模型，可接受 180 万像素的任意长宽比图像输入，实现了和 Gemini Pro 相近的场景文字识别能力以及和 GPT-4V 相匹的低幻觉率。 @@ -32,7 +33,9 @@ #### 📌 置顶 -* [2024.07.19] MiniCPM-Llama3-V 2.5 现已支持[vLLM](#vllm) ！ +* [2024.08.06] 🔥🔥🔥 我们开源了 MiniCPM-V 2.6，该模型在单图、多图和视频理解方面取得了优于 GPT-4V 的表现。我们还进一步提升了 MiniCPM-Llama3-V 2.5 的多项亮点能力，并首次支持了 iPad 上的实时视频理解。欢迎试用！ +* [2024.08.03] MiniCPM-Llama3-V 2.5 技术报告已发布！欢迎点击[这里](https://arxiv.org/abs/2408.01800)查看。 +* [2024.07.19] MiniCPM-Llama3-V 2.5 现已支持[vLLM](#vllm-部署-) ！ * [2024.05.28] 💥 MiniCPM-Llama3-V 2.5 现在在 llama.cpp 和 ollama 中完全支持其功能！**请拉取我们最新的 fork 来使用**：[llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) & [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5)。我们还发布了各种大小的 GGUF 版本，请点击[这里](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf/tree/main)查看。请注意，**目前官方仓库尚未支持 MiniCPM-Llama3-V 2.5**，我们也正积极推进将这些功能合并到 llama.cpp & ollama 官方仓库，敬请关注！ * [2024.05.28] 💫 我们现在支持 MiniCPM-Llama3-V 2.5 的 LoRA 微调，更多内存使用统计信息可以在[这里](https://github.com/OpenBMB/MiniCPM-V/tree/main/finetune#model-fine-tuning-memory-usage-statistics)找到。 * [2024.05.23] 🔍 我们添加了Phi-3-vision-128k-instruct 与 MiniCPM-Llama3-V 2.5的全面对比，包括基准测试评估、多语言能力和推理效率 🌟📊🌍🚀。点击[这里](./docs/compare_with_phi-3_vision.md)查看详细信息。 @@ -41,12 +44,14 @@ <br> +<details> +<summary>点击查看完整更新日志。</summary> * [2024.06.03] 现在，你可以利用多张低显存显卡（12G/16G）进行GPU串行推理。详情请参见该[文档](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md)配置。 * [2024.05.25] MiniCPM-Llama3-V 2.5 [支持流式输出和自定义系统提示词](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5#usage)了，欢迎试用! * [2024.05.24] 我们开源了 MiniCPM-Llama3-V 2.5 [gguf](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)，支持 [llama.cpp](#llamacpp-部署) 推理！实现端侧 6-8 tokens/s 的流畅解码，欢迎试用！ * [2024.05.20] 我们开源了 MiniCPM-Llama3-V 2.5，增强了 OCR 能力，支持 30 多种语言，并首次在端侧实现了 GPT-4V 级的多模态能力！我们提供了[高效推理](#手机端部署)和[简易微调](./finetune/readme.md)的支持，欢迎试用！ -* [2024.04.23] 我们增加了MiniCPM-V 2.0对 [vLLM](#vllm) 的支持，欢迎体验！ +* [2024.04.23] 我们增加了MiniCPM-V 2.0对 [vLLM](#vllm-部署-) 的支持，欢迎体验！ * [2024.04.18] 我们在 HuggingFace Space 新增了 MiniCPM-V 2.0 的 [demo](https://huggingface.co/spaces/openbmb/MiniCPM-V-2)，欢迎体验！ * [2024.04.17] MiniCPM-V 2.0 现在支持用户部署本地 [WebUI Demo](#本地webui-demo部署) 了，欢迎试用! * [2024.04.15] MiniCPM-V 2.0 现在可以通过 SWIFT 框架 [微调](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v-2最佳实践.md) 了，支持流式输出! @@ -54,29 +59,797 @@ * [2024.03.14] MiniCPM-V 现在支持 SWIFT 框架下的[微调](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v最佳实践.md)了，感谢 [Jintao](https://github.com/Jintao-Huang) 的贡献！ * [2024.03.01] MiniCPM-V 现在支持在 Mac 电脑上进行部署！ * [2024.02.01] 我们开源了 MiniCPM-V 和 OmniLMM-12B，分别可以支持高效的端侧部署和同规模领先的多模态能力！ +</details> ## 目录 <!-- omit in toc --> +- [MiniCPM-V 2.6](#minicpm-v-26) - [MiniCPM-Llama3-V 2.5](#minicpm-llama3-v-25) - [MiniCPM-V 2.0](#minicpm-v-20) -- [Demo](#demo) +- [Gradio Demo 🤗](#gradio-demo-) - [安装](#安装) - [推理](#推理) - [模型库](#模型库) - [多轮对话](#多轮对话) + - [多图理解](#多图理解) + - [少样本上下文学习](#少样本上下文学习) + - [视频理解](#视频理解) + - [多卡推理](#多卡推理) - [Mac 推理](#mac-推理) - [手机端部署](#手机端部署) - [本地WebUI Demo部署](#本地webui-demo部署) - [llama.cpp 部署](#llamacpp-部署) + - [ollama 部署](#ollama-部署) - [vLLM 部署 ](#vllm-部署-) - [微调](#微调) -- [未来计划](#未来计划) -- [🌟 Star History](#-star-history) -- [引用](#引用) +- [FAQs](#faqs) +## MiniCPM-V 2.6 + +**MiniCPM-V 2.6** 是 MiniCPM-V 系列中最新、性能最佳的模型。该模型基于 SigLip-400M 和 Qwen2-7B 构建，共 8B 参数。与 MiniCPM-Llama3-V 2.5 相比，MiniCPM-V 2.6 性能提升显著，并引入了多图和视频理解的新功能。MiniCPM-V 2.6 的主要特点包括： + + +- 🔥 **领先的性能。** + MiniCPM-V 2.6 在最新版本 OpenCompass 榜单上（综合 8 个主流多模态评测基准）平均得分 65.2，**以8B量级的大小在单图理解方面超越了 GPT-4o mini、GPT-4V、Gemini 1.5 Pro 和 Claude 3.5 Sonnet 等主流商用闭源多模态大模型**。 + +- 🖼️ **多图理解和上下文学习。** + MiniCPM-V 2.6 还支持**多图对话和推理**。它在 Mantis-Eval、BLINK、Mathverse mv 和 Sciverse mv 等主流多图评测基准中取得了**最佳水平**，并展现出了优秀的上下文学习能力。 + +- 🎬 **视频理解。** + MiniCPM-V 2.6 还可以**接受视频输入**，进行对话和提供涵盖时序和空间信息的详细视频描述。模型在 有/无字幕 评测场景下的 Video-MME 表现均超过了 **GPT-4V、Claude 3.5 Sonnet 和 LLaVA-NeXT-Video-34B**等商用闭源模型。 + +- 💪 **强大的 OCR 能力及其他功能。** + MiniCPM-V 2.6 可以处理任意长宽比的图像，像素数可达 180 万（如 1344x1344）。在 OCRBench 上取得**最佳水平，超过 GPT-4o、GPT-4V 和 Gemini 1.5 Pro 等商用闭源模型**。基于最新的 [RLAIF-V](https://github.com/RLHF-V/RLAIF-V/) 和 [VisCPM](https://github.com/OpenBMB/VisCPM) 技术，其具备了**可信的多模态行为**，在 Object HalBench 上的幻觉率显著低于 GPT-4o 和 GPT-4V，并支持英语、中文、德语、法语、意大利语、韩语等**多种语言**。 + +- 🚀 **卓越的效率。** + 除了对个人用户友好的模型大小，MiniCPM-V 2.6 还表现出**最先进的视觉 token 密度**（即每个视觉 token 编码的像素数量）。它**仅需 640 个 token 即可处理 180 万像素图像，比大多数模型少 75%**。这一特性优化了模型的推理速度、首 token 延迟、内存占用和功耗。因此，MiniCPM-V 2.6 可以支持 iPad 等终端设备上的高效**实时视频理解**。 + +- 💫 **易于使用。** + MiniCPM-V 2.6 可以通过多种方式轻松使用：(1) [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpmv-main/examples/llava/README-minicpmv2.6.md) 和 [ollama](https://github.com/OpenBMB/ollama/blob/minicpm-v2.6/examples/minicpm-v2.6/README.md) 支持在本地设备上进行高效的 CPU 推理，(2) [int4](https://huggingface.co/openbmb/MiniCPM-V-2_6-int4) 和 [GGUF](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) 格式的量化模型，有 16 种尺寸，(3) [vLLM](#vllm-部署-) 支持高吞吐量和内存高效的推理，(4) 针对新领域和任务进行微调，(5) 使用 [Gradio](#本地-webui-demo-) 快速设置本地 WebUI 演示，(6) 在线[demo](https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6)即可体验。 + +### 性能评估 <!-- omit in toc --> +<div align="center"> + <img src=assets/radar_final.png width=66% /> +</div> + +<details> +<summary>点击查看 OpenCompass, MME, MMVet, OCRBench, MMMU, MathVista, MMB, AI2D, TextVQA, DocVQA, HallusionBench, Object HalBench 上的单图评测结果详情。 </summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Token Density<sup>+</sup></th> + <th>OpenCompass</th> + <th>MME</th> + <th>MMVet</th> + <th>OCRBench</th> + <th>MMMU val</th> + <th>MathVista mini</th> + <th>MMB1.1 test</th> + <th>AI2D</th> + <th>TextVQA val</th> + <th>DocVQA test</th> + <th>HallusionBench</th> + <th>Object HalBench</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="15" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4o</td> + <td>-</td> + <td>1088</td> + <td>69.9</td> + <td>2328.7</td> + <td>69.1</td> + <td>736</td> + <td>69.2</td> + <td>61.3</td> + <td>82.2</td> + <td>84.6</td> + <td>-</td> + <td>92.8</td> + <td>55.0</td> + <td>17.6</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Claude 3.5 Sonnet</td> + <td>-</td> + <td>750</td> + <td>67.9</td> + <td>1920.0</td> + <td>66.0</td> + <td>788</td> + <td>65.9</td> + <td>61.6</td> + <td>78.5</td> + <td>80.2</td> + <td>-</td> + <td>95.2</td> + <td>49.9</td> + <td>13.8</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Gemini 1.5 Pro</td> + <td>-</td> + <td>-</td> + <td>64.4</td> + <td>2110.6</td> + <td>64.0</td> + <td>754</td> + <td>60.6</td> + <td>57.7</td> + <td>73.9</td> + <td>79.1</td> + <td>73.5</td> + <td>86.5</td> + <td>45.6</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4o mini</td> + <td>-</td> + <td>1088</td> + <td>64.1</td> + <td>2003.4</td> + <td>66.9</td> + <td>785</td> + <td>60.0</td> + <td>52.4</td> + <td>76.0</td> + <td>77.8</td> + <td>-</td> + <td>-</td> + <td>46.1</td> + <td>12.4</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>1088</td> + <td>63.5</td> + <td>2070.2</td> + <td>67.5</td> + <td>656</td> + <td>61.7</td> + <td>54.7</td> + <td>79.8</td> + <td>78.6</td> + <td>78.0</td> + <td>87.2</td> + <td>43.9</td> + <td>14.2</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Step-1V</td> + <td>-</td> + <td>-</td> + <td>59.5</td> + <td>2206.4</td> + <td>63.3</td> + <td>625</td> + <td>49.9</td> + <td>44.8</td> + <td>78.0</td> + <td>79.2</td> + <td>71.6</td> + <td>-</td> + <td>48.4</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Qwen-VL-Max</td> + <td>-</td> + <td>784</td> + <td>58.3</td> + <td>2281.7</td> + <td>61.8</td> + <td>684</td> + <td>52.0</td> + <td>43.4</td> + <td>74.6</td> + <td>75.7</td> + <td>79.5</td> + <td>93.1</td> + <td>41.2</td> + <td>13.4</td> + </tr> + <tr> + <td colspan="15" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Yi-34B</td> + <td>34B</td> + <td>157</td> + <td>55.0</td> + <td>2006.5</td> + <td>50.7</td> + <td>574</td> + <td>48.8</td> + <td>40.4</td> + <td>77.8</td> + <td>78.9</td> + <td>69.3</td> + <td>-</td> + <td>34.8</td> + <td>12.6</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Mini-Gemini-HD-34B</td> + <td>34B</td> + <td>157</td> + <td>-</td> + <td>2141</td> + <td>59.3</td> + <td>518</td> + <td>48.0</td> + <td>43.3</td> + <td>-</td> + <td>80.5</td> + <td>74.1</td> + <td>78.9</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Cambrian-34B</td> + <td>34B</td> + <td>1820</td> + <td>58.3</td> + <td>2049.9</td> + <td>53.2</td> + <td>591</td> + <td>50.4</td> + <td>50.3</td> + <td>77.8</td> + <td>79.5</td> + <td>76.7</td> + <td>75.5</td> + <td>41.6</td> + <td>14.7</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GLM-4V-9B</td> + <td>13B</td> + <td>784</td> + <td>59.1</td> + <td>2018.8</td> + <td>58.0</td> + <td>776</td> + <td>46.9</td> + <td>51.1</td> + <td>67.9</td> + <td>71.2</td> + <td>-</td> + <td>-</td> + <td>45.0</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>706</td> + <td>64.1</td> + <td>2215.1</td> + <td>54.3</td> + <td>794</td> + <td><strong>51.2</strong></td> + <td>58.3</td> + <td><strong>79.4</strong></td> + <td><strong>83.6</strong></td> + <td>77.4</td> + <td><strong>91.6</strong></td> + <td>45.0</td> + <td>21.3</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">MiniCPM-Llama-V 2.5</td> + <td>8B</td> + <td>1882</td> + <td>58.8</td> + <td>2024.6</td> + <td>52.8</td> + <td>725</td> + <td>45.8</td> + <td>54.3</td> + <td>72.0</td> + <td>78.4</td> + <td>76.6</td> + <td>84.8</td> + <td>42.4</td> + <td>10.3</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>2822</strong></td> + <td><strong>65.2</strong></td> + <td><strong>2348.4</strong>*</td> + <td><strong>60.0</strong></td> + <td><strong>852</strong>*</td> + <td>49.8*</td> + <td><strong>60.6</strong></td> + <td>78.0</td> + <td>82.1</td> + <td><strong>80.1<strong></td> + <td>90.8</td> + <td><strong>48.1</strong>*</td> + <td><strong>8.2</strong></td> + </tr> + </tbody> +</table> + +</div> +* 我们使用思维链提示词来评估这些基准。 + +<sup>+</sup> Token Density：每个视觉 token 在最大分辨率下编码的像素数，即最大分辨率下的像素数 / 视觉 token 数。 + +注意：闭源模型的 Token Density 由 API 收费方式估算得到。 +</details> + + +<details> +<summary>点击查看 Mantis Eval, BLINK, Mathverse mv, Sciverse mv, MIRB 上的多图评测结果详情。</summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Mantis Eval</th> + <th>BLINK val</th> + <th>Mathverse mv</th> + <th>Sciverse mv</th> + <th>MIRB</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="7" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>62.7</td> + <td>54.6</td> + <td>60.3</td> + <td>66.9</td> + <td>53.1</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Interleave-14B</td> + <td>14B</td> + <td>66.4</td> + <td>52.6</td> + <td>32.7</td> + <td>30.2</td> + <td>-</td> + </tr> + <tr> + <td colspan="7" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Emu2-Chat</td> + <td>37B</td> + <td>37.8</td> + <td>36.2</td> + <td>-</td> + <td>27.2</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">CogVLM</td> + <td>17B</td> + <td>45.2</td> + <td>41.1</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">VPG-C</td> + <td>7B</td> + <td>52.4</td> + <td>43.1</td> + <td>24.3</td> + <td>23.1</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">VILA 8B</td> + <td>8B</td> + <td>51.2</td> + <td>39.3</td> + <td>-</td> + <td>36.5</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternLM-XComposer-2.5</td> + <td>8B</td> + <td>53.1*</td> + <td>48.9</td> + <td>32.1*</td> + <td>-</td> + <td>42.5</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>59.0*</td> + <td>50.9</td> + <td>30.5*</td> + <td>34.4*</td> + <td><strong>56.9*</strong></td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>69.1</strong></td> + <td><strong>53.0</strong></td> + <td><strong>84.9</strong></td> + <td><strong>74.9</strong></td> + <td>53.8</td> + </tr> + </tbody> +</table> + + +</div> +* 正式开源模型权重的评测结果。 +</details> + +<details> +<summary>点击查看 Video-MME 和 Video-ChatGPT 上的视频评测结果详情。</summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th colspan="2">Video-MME</th> + <th colspan="5">Video-ChatGPT</th> + </tr> + <tr> + <th align="left"></th> + <th></th> + <th>w/o subs</th> + <th>w subs</th> + <th>Correctness</th> + <th>Detail</th> + <th>Context</th> + <th>Temporal</th> + <th>Consistency</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td colspan="9" align="left"><strong>Proprietary</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">Claude 3.5 Sonnet</td> + <td>-</td> + <td>60.0</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">GPT-4V</td> + <td>-</td> + <td>59.9</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td colspan="9" align="left"><strong>Open-source</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-7B</td> + <td>7B</td> + <td>-</td> + <td>-</td> + <td>3.39</td> + <td>3.29</td> + <td>3.92</td> + <td>2.60</td> + <td>3.12</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-34B</td> + <td>34B</td> + <td>-</td> + <td>-</td> + <td>3.29</td> + <td>3.23</td> + <td>3.83</td> + <td>2.51</td> + <td>3.47</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">CogVLM2-Video</td> + <td>12B</td> + <td>-</td> + <td>-</td> + <td>3.49</td> + <td><strong>3.46</strong></td> + <td>3.23</td> + <td><strong>2.98</strong></td> + <td><strong>3.64</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LongVA</td> + <td>7B</td> + <td>52.4</td> + <td>54.3</td> + <td>3.05</td> + <td>3.09</td> + <td>3.77</td> + <td>2.44</td> + <td><strong>3.64</strong></td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternVL2-8B</td> + <td>8B</td> + <td>54.0</td> + <td>56.9</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">InternLM-XComposer-2.5</td> + <td>8B</td> + <td>55.8</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + <td>-</td> + </tr> + <tr> + <td nowrap="nowrap" align="left">LLaVA-NeXT-Video</td> + <td>32B</td> + <td>60.2</td> + <td>63.0</td> + <td>3.48</td> + <td>3.37</td> + <td><strong>3.95</strong></td> + <td>2.64</td> + <td>3.28</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td nowrap="nowrap" align="left">MiniCPM-V 2.6</td> + <td>8B</td> + <td><strong>60.9</strong></td> + <td><strong>63.6</strong></td> + <td><strong>3.59</strong></td> + <td>3.28</td> + <td>3.93</td> + <td>2.73</td> + <td>3.62</td> + </tr> + </tbody> +</table> +</div> +</details> + + +<details> +<summary>点击查看 TextVQA, VizWiz, VQAv2, OK-VQA上的少样本评测结果详情。</summary> +<div align="center"> + +<table style="margin: 0px auto;"> + <thead> + <tr> + <th align="left">Model</th> + <th>Size</th> + <th>Shot</th> + <th>TextVQA val</th> + <th>VizWiz test-dev</th> + <th>VQAv2 test-dev</th> + <th>OK-VQA val</th> + </tr> + </thead> + <tbody align="center"> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">Flamingo</td> + <td rowspan="3">80B</td> + <td>0*</td> + <td>35.0</td> + <td>31.6</td> + <td>56.3</td> + <td>40.6</td> + </tr> + <tr> + <td>4</td> + <td>36.5</td> + <td>39.6</td> + <td>63.1</td> + <td><strong>57.4</strong></td> + </tr> + <tr> + <td>8</td> + <td>37.3</td> + <td>44.8</td> + <td>65.6</td> + <td>57.5</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">IDEFICS</td> + <td rowspan="3">80B</td> + <td>0*</td> + <td>30.9</td> + <td>36.0</td> + <td>60.0</td> + <td>45.2</td> + </tr> + <tr> + <td>4</td> + <td>34.3</td> + <td>40.4</td> + <td>63.6</td> + <td>52.4</td> + </tr> + <tr> + <td>8</td> + <td>35.7</td> + <td>46.1</td> + <td>64.8</td> + <td>55.1</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">OmniCorpus</td> + <td rowspan="3">7B</td> + <td>0*</td> + <td>43.0</td> + <td>49.8</td> + <td>63.2</td> + <td>45.5</td> + </tr> + <tr> + <td>4</td> + <td>45.4</td> + <td>51.3</td> + <td>64.5</td> + <td>46.5</td> + </tr> + <tr> + <td>8</td> + <td>45.6</td> + <td>52.2</td> + <td>64.7</td> + <td>46.6</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="3">Emu2</td> + <td rowspan="3">37B</td> + <td>0</td> + <td>26.4</td> + <td>40.4</td> + <td>33.5</td> + <td>26.7</td> + </tr> + <tr> + <td>4</td> + <td>48.2</td> + <td>54.6</td> + <td>67.0</td> + <td>53.2</td> + </tr> + <tr> + <td>8</td> + <td>49.3</td> + <td>54.7</td> + <td>67.8</td> + <td>54.1</td> + </tr> + <tr> + <td align="left" nowrap="nowrap" rowspan="2">MM1</td> + <td rowspan="2">30B</td> + <td>0</td> + <td>26.2</td> + <td>40.4</td> + <td>48.9</td> + <td>26.7</td> + </tr> + <tr> + <td>8</td> + <td>49.3</td> + <td>54.7</td> + <td><strong>70.9</strong></td> + <td>54.1</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td align="left" nowrap="nowrap" rowspan="3">MiniCPM-V 2.6<sup>+</sup></td> + <td rowspan="3">8B</td> + <td>0</td> + <td>43.9</td> + <td>33.8</td> + <td>45.4</td> + <td>23.9</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td>4</td> + <td>63.6</td> + <td>60.5</td> + <td>65.5</td> + <td>50.1</td> + </tr> + <tr style="background-color: #e6f2ff;"> + <td>8</td> + <td><strong>64.6</strong></td> + <td><strong>63.4</strong></td> + <td>68.2</td> + <td>51.4</td> + </tr> + </tbody> +</table> + + +</div> +* 使用 Flamingo 方式 zero image shot 和 two additional text shots 评估零样本性能。 + +<sup>+</sup> 我们在没有进行监督微调 (SFT) 的情况下评估预训练的模型权重 (ckpt)。 +</details> + +### 典型示例 <!-- omit in toc --> + +<div style="display: flex; flex-direction: column; align-items: center;"> + <img src="assets/minicpmv2_6/multi_img-bike.png" alt="Bike" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multi_img-menu.png" alt="Menu" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multi_img-code.png" alt="Code" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/ICL-Mem.png" alt="Mem" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multiling-medal.png" alt="medal" style="margin-bottom: 10px;"> +</div> +<details> + <summary>点击查看更多示例。</summary> + <div style="display: flex; flex-direction: column; align-items: center;"> + <img src="assets/minicpmv2_6/ICL-elec.png" alt="elec" style="margin-bottom: 5px;"> + <img src="assets/minicpmv2_6/multiling-olympic.png" alt="Menu" style="margin-bottom: 10px;"> + </div> +</details> + +我们将 MiniCPM-V 2.6 部署在iPad Pro上，并录制了以下演示视频。 + +<table align="center"> + <p align="center"> + <img src="assets/gif_cases/ai.gif" width=32%/> +      + <img src="assets/gif_cases/beer.gif" width=32%/> + </p> +</table> + +<table align="center"> + <p align="center"> + <video src="https://github.com/user-attachments/assets/21f4b818-ede1-4822-920e-91281725c830" width="360" /> </video> + <!-- <video src="https://github.com/user-attachments/assets/c835f757-206b-4d9c-8e36-70d67b453628" width="360" /> </video> --> + </p> +</table> ## MiniCPM-Llama3-V 2.5 + +<details> +<summary>查看 MiniCPM-Llama3-V 2.5 的详细信息</summary> + **MiniCPM-Llama3-V 2.5** 是 MiniCPM-V 系列的最新版本模型，基于 SigLip-400M 和 Llama3-8B-Instruct 构建，共 8B 参数量，相较于 MiniCPM-V 2.0 性能取得较大幅度提升。MiniCPM-Llama3-V 2.5 值得关注的特点包括： - 🔥 **领先的性能。** @@ -97,16 +870,6 @@ - 💫 **易于使用。** MiniCPM-Llama3-V 2.5 可以通过多种方式轻松使用：（1）[llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) 和 [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5) 支持在本地设备上进行高效的 CPU 推理；（2）提供 16 种尺寸的 [GGUF](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf) 格式量化模型；（3）仅需 2 张 V100 GPU 即可进行高效的 [LoRA](https://github.com/OpenBMB/MiniCPM-V/tree/main/finetune#lora-finetuning) 微调；（ 4）支持[流式输出](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5#usage)；（5）快速搭建 [Gradio](https://github.com/OpenBMB/MiniCPM-V/blob/main/web_demo_2.5.py) 和 [Streamlit](https://github.com/OpenBMB/MiniCPM-V/blob/main/web_demo_streamlit-2_5.py) 本地 WebUI demo；（ 6.）[HuggingFace Spaces](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) 交互式 demo。 -### MiniCPM-Llama3-V 2.5快速导航 <!-- omit in toc --> -你可以点击以下表格中的链接快速访问所需要的 MiniCPM-Llama3-V 2.5 功能。 - -| 功能分类 | | | | | | | || -|:--------:|:------:|:--------------:|:--------:|:-------:|:-----------:|:-----------:|:--------:|:-----------:| -| 推理 | [Transformers](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md) | [ollama](https://github.com/OpenBMB/ollama/tree/minicpm-v2.5/examples/minicpm-v2.5) | [SWIFT](./docs/swift_train_and_infer.md) | [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) | [Xinfrence](./docs/xinference_infer.md) | [Gradio](./web_demo_2.5.py) | [Streamlit](./web_demo_streamlit-2_5.py) |[vLLM](#vllm) -| 微调 | [Full-parameter](./finetune/readme.md) | [LoRA](./finetune/readme.md) | [SWIFT](./docs/swift_train_and_infer.md) | | | | | | -| 安卓部署 | [apk](http://minicpm.modelbest.cn/android/modelbest-release-20240528_182155.apk) | [llama.cpp](https://github.com/OpenBMB/llama.cpp/blob/minicpm-v2.5/examples/minicpmv/README.md) | | | | | | | -| 量化 | [Bnb](./quantize/bnb_quantize.py) | - ### 性能评估 <!-- omit in toc --> <div align="center"> @@ -403,20 +1166,8 @@ </p> </table> -我们将 MiniCPM-Llama3-V 2.5 部署在小米 14 Pro 上，并录制了以下演示视频。 -<table align="center"> - <p align="center"> - <img src="assets/gif_cases/ticket.gif" width=32%/> - <img src="assets/gif_cases/meal_plan.gif" width=32%/> - </p> -</table> - -<table align="center"> - <p align="center" width=80%> - <img src="assets/gif_cases/1-4.gif" width=72%/> - </p> -</table> +</details> ## MiniCPM-V 2.0 @@ -481,13 +1232,13 @@ | OmniLMM-12B | [文档](./omnilmm.md) | -## Demo +## Gradio Demo 🤗 -我们提供由 Hugging Face [Gradio](https://github.com/gradio-app/gradio) 支持的在线和本地 Demo。Gradio 是目前最流行的模型部署框架，支持流式输出、进度条、process bars 和其他常用功能。 +我们提供由 Hugging Face Gradio <a href='https://github.com/gradio-app/gradio'><img src='https://img.shields.io/github/stars/gradio-app/gradio'></a> 支持的在线和本地 Demo。Gradio 是目前最流行的模型部署框架，支持流式输出、进度条、process bars 和其他常用功能。 ### Online Demo <!-- omit in toc --> -欢迎试用 Hugging Face Spaces 上的 [MiniCPM-Llama3-V 2.5](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) ｜ [MiniCPM-V 2.0](https://huggingface.co/spaces/openbmb/MiniCPM-V-2) Online Demo。 +欢迎试用 Online Demo: [MiniCPM-V 2.6](https://huggingface.co/spaces/openbmb/MiniCPM-V-2_6) | [MiniCPM-Llama3-V 2.5](https://huggingface.co/spaces/openbmb/MiniCPM-Llama3-V-2_5) | [MiniCPM-V 2.0](https://huggingface.co/spaces/openbmb/MiniCPM-V-2) 。 ### 本地 WebUI Demo <!-- omit in toc --> @@ -499,10 +1250,8 @@ pip install -r requirements.txt ```shell # 对于 NVIDIA GPU，请运行： -python web_demo_2.5.py --device cuda +python web_demo_2.6.py --device cuda -# 对于搭载 MPS 的 Mac（Apple 芯片或 AMD GPU），请运行： -PYTORCH_ENABLE_MPS_FALLBACK=1 python web_demo_2.5.py --device mps ``` @@ -535,8 +1284,11 @@ pip install -r requirements.txt | 模型 | 设备 | 资源 |          简介 | 下载链接 | |:--------------|:-:|:----------:|:-------------------|:---------------:| -| MiniCPM-Llama3-V 2.5| GPU | 19 GB | 最新版本，提供最佳的端侧多模态理解能力。 | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5) | -| MiniCPM-Llama3-V 2.5 gguf| CPU | 5 GB | gguf 版本，更低的内存占用和更高的推理效率。 | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-gguf) | +| MiniCPM-V 2.6| GPU | 17 GB | 最新版本，提供最佳的端侧单图、多图、视频理解能力。 | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6) | +| MiniCPM-V 2.6 gguf | CPU | 6 GB | gguf 版本，更低的内存占用和更高的推理效率。 | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6-gguf) | +| MiniCPM-V 2.6 int4 | GPU | 7 GB | int4量化版，更低显存占用。 | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2_6-int4)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6-int4) | +| MiniCPM-Llama3-V 2.5| GPU | 19 GB | 提供出色的端侧多模态理解能力。 | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5) | +| MiniCPM-Llama3-V 2.5 gguf | CPU | 6 GB | gguf 版本，更低的内存占用和更高的推理效率。 | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-gguf) | | MiniCPM-Llama3-V 2.5 int4 | GPU | 8 GB | int4量化版，更低显存占用。 | [🤗](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-int4/)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5-int4) | | MiniCPM-V 2.0 | GPU | 8 GB | 轻量级版本，平衡计算开销和多模态理解能力。 | [🤗](https://huggingface.co/openbmb/MiniCPM-V-2)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2) | | MiniCPM-V 1.0 | GPU | 7 GB | 最轻量版本， 提供最快的推理速度。 | [🤗](https://huggingface.co/openbmb/MiniCPM-V)    [<img src="./assets/modelscope_logo.png" width="20px"></img>](https://modelscope.cn/models/OpenBMB/MiniCPM-V) | @@ -553,30 +1305,40 @@ pip install -r requirements.txt ```python -from chat import MiniCPMVChat, img2base64 import torch -import json +from PIL import Image +from transformers import AutoModel, AutoTokenizer torch.manual_seed(0) -chat_model = MiniCPMVChat('openbmb/MiniCPM-Llama3-V-2_5') +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) -im_64 = img2base64('./assets/airplane.jpeg') +image = Image.open('./assets/airplane.jpeg').convert('RGB') # First round chat -msgs = [{"role": "user", "content": "Tell me the model of this aircraft."}] +question = "Tell me the model of this aircraft." +msgs = [{'role': 'user', 'content': [image, question]}] -inputs = {"image": im_64, "question": json.dumps(msgs)} -answer = chat_model.chat(inputs) +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) print(answer) # Second round chat # pass history context of multi-turn conversation -msgs.append({"role": "assistant", "content": answer}) -msgs.append({"role": "user", "content": "Introduce something about Airbus A380."}) +msgs.append({"role": "assistant", "content": [answer]}) +msgs.append({"role": "user", "content": ["Introduce something about Airbus A380."]}) -inputs = {"image": im_64, "question": json.dumps(msgs)} -answer = chat_model.chat(inputs) +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) print(answer) ``` @@ -588,6 +1350,127 @@ print(answer) "The Airbus A380 is a double-deck, wide-body, four-engine jet airliner made by Airbus. It is the world's largest passenger airliner and is known for its long-haul capabilities. The aircraft was developed to improve efficiency and comfort for passengers traveling over long distances. It has two full-length passenger decks, which can accommodate more passengers than a typical single-aisle airplane. The A380 has been operated by airlines such as Lufthansa, Singapore Airlines, and Emirates, among others. It is widely recognized for its unique design and significant impact on the aviation industry." ``` +#### 多图理解 +<details> +<summary> 点击查看使用 MiniCPM-V 2.6 进行多图理解的Python示例 </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +image1 = Image.open('image1.jpg').convert('RGB') +image2 = Image.open('image2.jpg').convert('RGB') +question = 'Compare image 1 and image 2, tell me about the differences between image 1 and image 2.' + +msgs = [{'role': 'user', 'content': [image1, image2, question]}] + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) +print(answer) +``` +</details> + +#### 少样本上下文学习 + +<details> +<summary> 点击查看使用 MiniCPM-V 2.6 进行few-shot推理的Python示例 </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +question = "production date" +image1 = Image.open('example1.jpg').convert('RGB') +answer1 = "2023.08.04" +image2 = Image.open('example2.jpg').convert('RGB') +answer2 = "2007.04.24" +image_test = Image.open('test.jpg').convert('RGB') + +msgs = [ + {'role': 'user', 'content': [image1, question]}, {'role': 'assistant', 'content': [answer1]}, + {'role': 'user', 'content': [image2, question]}, {'role': 'assistant', 'content': [answer2]}, + {'role': 'user', 'content': [image_test, question]} +] + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer +) +print(answer) +``` +</details> + +#### 视频理解 +<details> +<summary> 点击查看使用 MiniCPM-V 2.6 进行视频理解的Python示例 </summary> + +```python +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer +from decord import VideoReader, cpu # pip install decord + +model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager +model = model.eval().cuda() +tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) + +MAX_NUM_FRAMES=64 # if cuda OOM set a smaller number + +def encode_video(video_path): + def uniform_sample(l, n): + gap = len(l) / n + idxs = [int(i * gap + gap / 2) for i in range(n)] + return [l[i] for i in idxs] + + vr = VideoReader(video_path, ctx=cpu(0)) + sample_fps = round(vr.get_avg_fps() / 1) # FPS + frame_idx = [i for i in range(0, len(vr), sample_fps)] + if len(frame_idx) > MAX_NUM_FRAMES: + frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES) + frames = vr.get_batch(frame_idx).asnumpy() + frames = [Image.fromarray(v.astype('uint8')) for v in frames] + print('num frames:', len(frames)) + return frames + +video_path="video_test.mp4" +frames = encode_video(video_path) +question = "Describe the video" +msgs = [ + {'role': 'user', 'content': frames + [question]}, +] + +# Set decode params for video +params = {} +params["use_image_id"] = False +params["max_slice_nums"] = 2 # 如果cuda OOM且视频分辨率大于448*448可设为1 + +answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + **params +) +print(answer) +``` +</details> + ### 多卡推理 您可以通过将模型的层分布在多个低显存显卡（12 GB 或 16 GB）上，运行 MiniCPM-Llama3-V 2.5。请查看该[教程](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/inference_on_multiple_gpus.md)，详细了解如何使用多张低显存显卡载入模型并进行推理。 @@ -630,7 +1513,7 @@ PYTORCH_ENABLE_MPS_FALLBACK=1 python test.py ### 手机端部署 -MiniCPM-Llama3-V 2.5 和 MiniCPM-V 2.0 可运行在Android手机上，点击[MiniCPM-Llama3-V 2.5](http://minicpm.modelbest.cn/android/modelbest-release-20240528_182155.apk) / [MiniCPM-V 2.0](https://github.com/OpenBMB/mlc-MiniCPM)安装apk使用; +MiniCPM-V 2.0 可运行在Android手机上，点击[MiniCPM-V 2.0](https://github.com/OpenBMB/mlc-MiniCPM)安装apk使用; ### 本地WebUI Demo部署 <details> @@ -642,38 +1525,101 @@ pip install -r requirements.txt ```shell # For NVIDIA GPUs, run: -python web_demo_2.5.py --device cuda - -# For Mac with MPS (Apple silicon or AMD GPUs), run: -PYTORCH_ENABLE_MPS_FALLBACK=1 python web_demo_2.5.py --device mps +python web_demo_2.6.py --device cuda ``` </details> ### llama.cpp 部署<a id="llamacpp-部署"></a> -MiniCPM-Llama3-V 2.5 现在支持llama.cpp啦! 用法请参考我们的fork [llama.cpp](https://github.com/OpenBMB/llama.cpp/tree/minicpm-v2.5/examples/minicpmv)， 在手机上可以支持 6~8 token/s 的流畅推理（测试环境：Xiaomi 14 pro + Snapdragon 8 Gen 3）。 +MiniCPM-V 2.6 现在支持llama.cpp啦! 用法请参考[我们的fork llama.cpp](https://github.com/OpenBMB/llama.cpp/tree/minicpmv-main/examples/llava/README-minicpmv2.6.md)， 在iPad上可以支持 16~18 token/s 的流畅推理（测试环境：iPad Pro + M4）。 + +### ollama 部署<a id="ollama-部署"></a> +MiniCPM-V 2.6 现在支持ollama啦! 用法请参考[我们的fork ollama](https://github.com/OpenBMB/ollama/blob/minicpm-v2.6/examples/minicpm-v2.6/README.md)， 在iPad上可以支持 16~18 token/s 的流畅推理（测试环境：iPad Pro + M4）。 ### vLLM 部署 <a id='vllm'></a> <details> -<summary>点击查看, vLLM 现已官方支持MiniCPM-V 2.0 和 MiniCPM-Llama3-V 2.5</summary> +<summary>点击查看, vLLM 现已官方支持MiniCPM-V 2.6、MiniCPM-Llama3-V 2.5 和 MiniCPM-V 2.0 </summary> -1. 首先克隆官方的 vLLM 库: +1. 安装 vLLM(>=0.5.4): ```shell -git clone https://github.com/vllm-project/vllm.git +pip install vllm ``` -2. 安装 vLLM 库: -```shell -cd vllm -pip install -e . -``` -3. 安装 timm 库: +3. 安装 timm 库: （可选，MiniCPM-V 2.0需安装） ```shell pip install timm=0.9.10 ``` -4. 运行示例代码:（如果使用本地路径的模型，请确保模型代码已更新到Hugging Face上的最新版) -```shell -python examples/minicpmv_example.py -``` +4. 运行示例代码:（注意：如果使用本地路径的模型，请确保模型代码已更新到Hugging Face上的最新版) +```python +from transformers import AutoTokenizer +from PIL import Image +from vllm import LLM, SamplingParams +MODEL_NAME = "openbmb/MiniCPM-V-2_6" +# Also available for previous models +# MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5" +# MODEL_NAME = "HwwwH/MiniCPM-V-2" + +image = Image.open("xxx.png").convert("RGB") +tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) +llm = LLM( + model=MODEL_NAME, + trust_remote_code=True, + gpu_memory_utilization=1, + max_model_len=2048 +) + +messages = [{ + "role": + "user", + "content": + # Number of images + "(<image>./</image>)" + \ + "\nWhat is the content of this image?" +}] +prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True +) + +# Single Inference +inputs = { + "prompt": prompt, + "multi_modal_data": { + "image": image + # Multi images, the number of images should be equal to that of `(<image>./</image>)` + # "image": [image, image] + }, +} +# Batch Inference +# inputs = [{ +# "prompt": prompt, +# "multi_modal_data": { +# "image": image +# }, +# } for _ in 2] + + +# 2.6 +stop_tokens = ['<|im_end|>', '<|endoftext|>'] +stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] +# 2.0 +# stop_token_ids = [tokenizer.eos_id] +# 2.5 +# stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id] + +sampling_params = SamplingParams( + stop_token_ids=stop_token_ids, + use_beam_search=True, + temperature=0, + best_of=3, + max_tokens=1024 +) + +outputs = llm.generate(inputs, sampling_params=sampling_params) + +print(outputs[0].outputs[0].text) +``` +4. [点击此处](https://modelbest.feishu.cn/wiki/C2BWw4ZP0iCDy7kkCPCcX2BHnOf?from=from_copylink)查看带视频推理和其他有关 `vLLM` 的信息。 </details> @@ -692,11 +1638,8 @@ python examples/minicpmv_example.py 参考文档：[MiniCPM-V 1.0](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v最佳实践.md)，[MiniCPM-V 2.0](https://github.com/modelscope/swift/blob/main/docs/source/Multi-Modal/minicpm-v-2最佳实践.md) -## 未来计划 - -- [x] 支持 MiniCPM-V 系列模型微调 -- [ ] 实时多模态交互代码开源 - +## FAQs +点击查看 [FAQs](./docs/faqs.md) ## 模型协议 <!-- omit in toc --> @@ -720,13 +1663,7 @@ python examples/minicpmv_example.py - <img src="assets/modelbest.png" width="28px"> [面壁智能](https://modelbest.cn/) - <img src="assets/zhihu.webp" width="28px"> [知乎](https://www.zhihu.com/ ) -## 其他多模态项目 <!-- omit in toc --> - -👏 欢迎了解我们更多的多模态项目： - -[VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) - -## 🌟 Star History +## 🌟 Star History <!-- omit in toc --> <table align="center"> @@ -754,34 +1691,23 @@ python examples/minicpmv_example.py /> </picture> --> +## 支持技术和其他多模态项目 <!-- omit in toc --> -## 引用 +👏 欢迎了解 MiniCPM-V 背后的支持技术和更多我们的多模态项目！ + +[VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) + + + +## 引用 <!-- omit in toc --> 如果您觉得我们模型/代码/论文有帮助，请给我们 ⭐ 和 引用 📝，感谢！ ```bib -@article{yu2023rlhf, - title={Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback}, - author={Yu, Tianyu and Yao, Yuan and Zhang, Haoye and He, Taiwen and Han, Yifeng and Cui, Ganqu and Hu, Jinyi and Liu, Zhiyuan and Zheng, Hai-Tao and Sun, Maosong and others}, - journal={arXiv preprint arXiv:2312.00849}, - year={2023} -} -@article{viscpm, - title={Large Multilingual Models Pivot Zero-Shot Multimodal Learning across Languages}, - author={Jinyi Hu and Yuan Yao and Chongyi Wang and Shan Wang and Yinxu Pan and Qianyu Chen and Tianyu Yu and Hanghao Wu and Yue Zhao and Haoye Zhang and Xu Han and Yankai Lin and Jiao Xue and Dahai Li and Zhiyuan Liu and Maosong Sun}, - journal={arXiv preprint arXiv:2308.12038}, - year={2023} -} -@article{xu2024llava-uhd, - title={{LLaVA-UHD}: an LMM Perceiving Any Aspect Ratio and High-Resolution Images}, - author={Xu, Ruyi and Yao, Yuan and Guo, Zonghao and Cui, Junbo and Ni, Zanlin and Ge, Chunjiang and Chua, Tat-Seng and Liu, Zhiyuan and Huang, Gao}, - journal={arXiv preprint arXiv:2403.11703}, - year={2024} -} -@article{yu2024rlaifv, - title={RLAIF-V: Aligning MLLMs through Open-Source AI Feedback for Super GPT-4V Trustworthiness}, - author={Yu, Tianyu and Zhang, Haoye and Yao, Yuan and Dang, Yunkai and Chen, Da and Lu, Xiaoman and Cui, Ganqu and He, Taiwen and Liu, Zhiyuan and Chua, Tat-Seng and Sun, Maosong}, - journal={arXiv preprint arXiv:2405.17220}, +@article{yao2024minicpm, + title={MiniCPM-V: A GPT-4V Level MLLM on Your Phone}, + author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others}, + journal={arXiv preprint arXiv:2408.01800}, year={2024} } ``` diff --git a/assets/gif_cases/ai.gif b/assets/gif_cases/ai.gif new file mode 100644 index 0000000..d511496 Binary files /dev/null and b/assets/gif_cases/ai.gif differ diff --git a/assets/gif_cases/beer.gif b/assets/gif_cases/beer.gif new file mode 100644 index 0000000..a295399 Binary files /dev/null and b/assets/gif_cases/beer.gif differ diff --git a/assets/gif_cases/mb.gif b/assets/gif_cases/mb.gif new file mode 100644 index 0000000..dce6274 Binary files /dev/null and b/assets/gif_cases/mb.gif differ diff --git a/assets/gif_cases/rabbit.gif b/assets/gif_cases/rabbit.gif new file mode 100644 index 0000000..d27d6a4 Binary files /dev/null and b/assets/gif_cases/rabbit.gif differ diff --git a/assets/gif_cases/ticket.gif b/assets/gif_cases/ticket.gif index b09d831..28ef137 100644 Binary files a/assets/gif_cases/ticket.gif and b/assets/gif_cases/ticket.gif differ diff --git a/assets/gif_cases/wfh.gif b/assets/gif_cases/wfh.gif new file mode 100644 index 0000000..7ce8440 Binary files /dev/null and b/assets/gif_cases/wfh.gif differ diff --git a/assets/gif_cases/zoo.gif b/assets/gif_cases/zoo.gif new file mode 100644 index 0000000..707b86e Binary files /dev/null and b/assets/gif_cases/zoo.gif differ diff --git a/assets/minicpm-v23.png b/assets/minicpm-v23.png new file mode 100644 index 0000000..34c912f Binary files /dev/null and b/assets/minicpm-v23.png differ diff --git a/assets/minicpm-v24.png b/assets/minicpm-v24.png new file mode 100644 index 0000000..7b3cccc Binary files /dev/null and b/assets/minicpm-v24.png differ diff --git a/assets/minicpmv22.jpeg b/assets/minicpmv22.jpeg new file mode 100644 index 0000000..a911a6b Binary files /dev/null and b/assets/minicpmv22.jpeg differ diff --git a/assets/minicpmv2_6/ICL-Mem.png b/assets/minicpmv2_6/ICL-Mem.png new file mode 100644 index 0000000..e9517ba Binary files /dev/null and b/assets/minicpmv2_6/ICL-Mem.png differ diff --git a/assets/minicpmv2_6/ICL-elec.png b/assets/minicpmv2_6/ICL-elec.png new file mode 100644 index 0000000..de32d77 Binary files /dev/null and b/assets/minicpmv2_6/ICL-elec.png differ diff --git a/assets/minicpmv2_6/multi_img-bike.png b/assets/minicpmv2_6/multi_img-bike.png new file mode 100644 index 0000000..c19975d Binary files /dev/null and b/assets/minicpmv2_6/multi_img-bike.png differ diff --git a/assets/minicpmv2_6/multi_img-code.png b/assets/minicpmv2_6/multi_img-code.png new file mode 100644 index 0000000..abc4a6c Binary files /dev/null and b/assets/minicpmv2_6/multi_img-code.png differ diff --git a/assets/minicpmv2_6/multi_img-menu.png b/assets/minicpmv2_6/multi_img-menu.png new file mode 100644 index 0000000..2a0ba36 Binary files /dev/null and b/assets/minicpmv2_6/multi_img-menu.png differ diff --git a/assets/minicpmv2_6/multiling-medal.png b/assets/minicpmv2_6/multiling-medal.png new file mode 100644 index 0000000..af75d3d Binary files /dev/null and b/assets/minicpmv2_6/multiling-medal.png differ diff --git a/assets/minicpmv2_6/multiling-olympic.png b/assets/minicpmv2_6/multiling-olympic.png new file mode 100644 index 0000000..3c869e8 Binary files /dev/null and b/assets/minicpmv2_6/multiling-olympic.png differ diff --git a/assets/radar_final.png b/assets/radar_final.png new file mode 100644 index 0000000..ac60085 Binary files /dev/null and b/assets/radar_final.png differ diff --git a/chat.py b/chat.py index 8dbf8ef..8d36e56 100644 --- a/chat.py +++ b/chat.py @@ -183,13 +183,87 @@ class MiniCPMV2_5: ) return answer +class MiniCPMV2_6: + def __init__(self, model_path, multi_gpus=False) -> None: + + print('torch_version:', torch.__version__) + if multi_gpus: # inference on multi-gpus + from accelerate import load_checkpoint_and_dispatch, init_empty_weights, infer_auto_device_map + with init_empty_weights(): + model = AutoModel.from_pretrained(model_path, trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) + + device_map = infer_auto_device_map(model, max_memory={0: "10GB", 1: "10GB"}, + no_split_module_classes=['SiglipVisionTransformer', 'Qwen2DecoderLayer']) + device_id = device_map["llm.model.embed_tokens"] + device_map["llm.lm_head"] = device_id # first and last layer of llm should be in the same device + device_map["vpm"] = device_id + device_map["resampler"] = device_id + device_id2 = device_map["llm.model.layers.26"] + device_map["llm.model.layers.8"] = device_id2 + device_map["llm.model.layers.9"] = device_id2 + device_map["llm.model.layers.10"] = device_id2 + device_map["llm.model.layers.11"] = device_id2 + device_map["llm.model.layers.12"] = device_id2 + device_map["llm.model.layers.13"] = device_id2 + device_map["llm.model.layers.14"] = device_id2 + device_map["llm.model.layers.15"] = device_id2 + device_map["llm.model.layers.16"] = device_id2 + print(device_map) + + self.model = load_checkpoint_and_dispatch(model, model_path, dtype=torch.bfloat16, device_map=device_map) + self.model.eval() + else: + self.model = AutoModel.from_pretrained(model_path, trust_remote_code=True, + attn_implementation='sdpa', torch_dtype=torch.bfloat16) + self.model.eval().cuda() + + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + + def chat(self, input): + image = None + if "image" in input and len(input["image"]) > 10: # legacy API + try: + image = Image.open(io.BytesIO(base64.b64decode(input['image']))).convert('RGB') + except Exception as e: + return "Image decode error" + + msgs = json.loads(input["question"]) + + for msg in msgs: + contents = msg.pop('content') # support str or List[Dict] + if isinstance(contents, str): + contents = [contents] + + new_cnts = [] + for c in contents: + if isinstance(c, dict): + if c['type'] == 'text': + c = c['pairs'] + elif c['type'] == 'image': + c = Image.open(io.BytesIO(base64.b64decode(c["pairs"]))).convert('RGB') + else: + raise ValueError("content type only support text and image.") + new_cnts.append(c) + msg['content'] = new_cnts + print(f'msgs: {str(msgs)}') + + answer = self.model.chat( + image=image, + msgs=msgs, + tokenizer=self.tokenizer, + ) + return answer + class MiniCPMVChat: - def __init__(self, model_path) -> None: + def __init__(self, model_path, multi_gpus=False) -> None: if '12B' in model_path: self.model = OmniLMM12B(model_path) elif 'MiniCPM-Llama3-V' in model_path: self.model = MiniCPMV2_5(model_path) + elif 'MiniCPM-V-2_6' in model_path: + self.model = MiniCPMV2_6(model_path, multi_gpus) else: self.model = MiniCPMV(model_path) diff --git a/docs/MiniCPM_Llama3_V_25_technical_report.pdf b/docs/MiniCPM_Llama3_V_25_technical_report.pdf new file mode 100644 index 0000000..91a2d9d Binary files /dev/null and b/docs/MiniCPM_Llama3_V_25_technical_report.pdf differ diff --git a/docs/faqs.md b/docs/faqs.md new file mode 100644 index 0000000..4cae555 --- /dev/null +++ b/docs/faqs.md @@ -0,0 +1,30 @@ +### FAQs + +<details> +<summary>Q: How to choose between sampling or beam search for inference </summary> + +In various scenarios, the quality of results obtained from beam search and sampling decoding strategies can vary. You can determine your decoding strategy based on the following aspects: + +If you have the following needs, consider using sampling decoding: + +1. You require faster inference speed. +2. You wish for a streaming generation approach. +3. Your task necessitates some open-ended responses. + +If your task is about providing deterministic answers, you might want to experiment with beam search to see if it can achieve better outcomes. +</details> + + +<details> +<summary>Q: How to ensure that the model generates results of sufficient length</summary> + +We've observed that during multi-language inference on MiniCPM-V 2.6, the generation sometimes ends prematurely. You can improve the results by passing a `min_new_tokens` parameter. +```python +res = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + min_new_tokens=100 +) +``` +</details> diff --git a/docs/wechat.md b/docs/wechat.md index bb46158..cdc0bf9 100644 --- a/docs/wechat.md +++ b/docs/wechat.md @@ -1,5 +1,5 @@ <div align="center"> -<img src="../assets/minicpm-v22.png" width="60%"/> +<img src="../assets/minicpm-v24.png" width="60%"/> <p> 扫码加入「MiniCPM-V 交流群」 </p> <p> Scan the QR code to join the "MiniCPM-V Discussion Group" </p> diff --git a/finetune/dataset.py b/finetune/dataset.py index 92807c3..38d05d4 100644 --- a/finetune/dataset.py +++ b/finetune/dataset.py @@ -105,7 +105,7 @@ def data_collator(examples, padding_value=0, max_length=2048): } -def conversation_to_ids(conversation, tokenizer, llm_type=None): +def conversation_to_ids(conversation, tokenizer, llm_type=None, new_schema=False): """ for single image multi-turn conversation conversation: [{'role': 'user', 'content': 'Describe this image'}, @@ -115,6 +115,10 @@ def conversation_to_ids(conversation, tokenizer, llm_type=None): input_ids, context, raw_msg = conversation_to_ids_llama3( conversation, tokenizer ) + elif llm_type == "qwen2": + input_ids, context, raw_msg = conversation_to_ids_qwen2( + conversation, tokenizer + ) else: input_ids, context, raw_msg = conversation_to_ids_minicpm( conversation, tokenizer @@ -125,6 +129,7 @@ def conversation_to_ids(conversation, tokenizer, llm_type=None): # build target target = torch.full_like(ids, -100, dtype=torch.int32) + for i in range(1, len(ids)): if context[i] == 0: target[i - 1] = ids[i] @@ -133,14 +138,21 @@ def conversation_to_ids(conversation, tokenizer, llm_type=None): target[i - 1] = tokenizer.eot_id else: target[i - 1] = tokenizer.eos_id - + # build image bound - image_start_tokens = torch.where(ids == tokenizer.im_start_id)[0] - image_start_tokens += 1 - image_end_tokens = torch.where(ids == tokenizer.im_end_id)[0] + if new_schema: + start_cond = (ids == tokenizer.im_start_id) | (ids == tokenizer.slice_start_id) + end_cond = (ids == tokenizer.im_end_id) | (ids == tokenizer.slice_end_id) + image_start_tokens = torch.where(start_cond)[0] + image_start_tokens += 1 + image_end_tokens = torch.where(end_cond)[0] + else: + image_start_tokens = torch.where(ids == tokenizer.im_start_id)[0] + image_start_tokens += 1 + image_end_tokens = torch.where(ids == tokenizer.im_end_id)[0] if len(image_start_tokens) != len(image_end_tokens): print("image start token != image end tokens") - + if len(image_start_tokens) > 0: image_bound = torch.hstack( [image_start_tokens.unsqueeze(-1), image_end_tokens.unsqueeze(-1)] @@ -230,6 +242,46 @@ def conversation_to_ids_llama3(conversation, tokenizer): return input_ids, context, raw_msg +def conversation_to_ids_qwen2(conversation, tokenizer): + raw_msg = "" + chat = [] + context = [] + for idx, msg in enumerate(conversation): + role = msg["role"] + message = msg["content"] + assert role in ["user", "assistant"] + if role == "user": + prefix = "user" + else: + prefix = "assistant" + chat.append({"role":prefix, "content":message}) + raw_msg += prefix + message + assert set([i['role'] for i in chat]) & set(['assistant']) + + ret = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=False) + input_ids = tokenizer.apply_chat_template(chat, tokenize=True, add_generation_prompt=False) + input_ids = np.array(input_ids) + + start_idxs = np.where(input_ids == tokenizer.convert_tokens_to_ids('<|im_start|>'))[0] + assistant_idxs = np.where(input_ids == tokenizer.convert_tokens_to_ids('assistant'))[0] + end_idxs = np.where(input_ids == tokenizer.convert_tokens_to_ids('<|im_end|>'))[0] + + context = np.ones_like(input_ids, dtype=np.int8) + + for assistant_idx in assistant_idxs: + if assistant_idx-1 in set(start_idxs): + st = assistant_idx + 1 + for end_idx in end_idxs: + if end_idx > st: + context[st: end_idx + 1] = 0 + break + + input_ids = np.hstack(input_ids) + context = np.hstack(context) + return input_ids, context, raw_msg + + + def preprocess( image, conversation, @@ -256,8 +308,14 @@ def preprocess( default_image_placeholder = ( tokenizer.im_start + tokenizer.unk_token * query_nums + tokenizer.im_end ) + new_schema = False + use_image_id = False + if llm_type=='qwen2': + new_schema = True + use_image_id = True if slice_config: images = [] + image_id_cnt = 0 source_image, patches, best_grid = slice_image( image, slice_config["max_slice_nums"], @@ -270,9 +328,11 @@ def preprocess( for i in range(len(patches)): for j in range(len(patches[0])): images.append(patches[i][j]) - + if use_image_id: + image_placeholder = f'{tokenizer.im_id_start}{image_id_cnt}{tokenizer.im_id_end}' + image_placeholder + image_id_cnt += 1 image_placeholder += get_grid_placeholder( - tokenizer, best_grid, query_nums) + tokenizer, best_grid, query_nums, new_schema = new_schema) images = [transform(i) for i in images] else: images = [transform(image)] @@ -286,7 +346,7 @@ def preprocess( image_placeholder + "\n" + conversation[0]["content"] ) - input_dict = conversation_to_ids(conversation, tokenizer, llm_type) + input_dict = conversation_to_ids(conversation, tokenizer, llm_type, new_schema) if batch_vision: tgt_sizes = [] @@ -424,7 +484,7 @@ def split_to_patches(image, grid): return patches -def get_grid_placeholder(tokenizer, grid, query_num): +def get_grid_placeholder(tokenizer, grid, query_num, new_schema=False): image_placeholder = ( tokenizer.im_start + tokenizer.unk_token * query_num + tokenizer.im_end ) @@ -437,7 +497,10 @@ def get_grid_placeholder(tokenizer, grid, query_num): for j in range(cols): lines.append(image_placeholder) slices.append("".join(lines)) - slice_placeholder = tokenizer.slice_start + \ + if new_schema: + slice_placeholder = '\n'.join(slices) + else: + slice_placeholder = tokenizer.slice_start + \ "\n".join(slices) + tokenizer.slice_end return slice_placeholder diff --git a/finetune/finetune.py b/finetune/finetune.py index 1c42a17..04cf2eb 100644 --- a/finetune/finetune.py +++ b/finetune/finetune.py @@ -6,6 +6,8 @@ from dataclasses import dataclass, field from functools import partial from typing import Dict, List, Optional, Union, Literal, Tuple from types import MethodType +from torchvision import transforms + import torch import transformers from accelerate.utils import DistributedType @@ -130,6 +132,18 @@ def make_supervised_data_module( ) +def build_transform(): + IMAGENET_INCEPTION_MEAN = (0.5, 0.5, 0.5) # timm.data.IMAGENET_INCEPTION_MEAN + IMAGENET_INCEPTION_STD = (0.5, 0.5, 0.5) # timm.data.IMAGENET_INCEPTION_STD + return transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize( + mean=IMAGENET_INCEPTION_MEAN, std=IMAGENET_INCEPTION_STD + ), + ] + ) + def get_parameter_number(model): trainable_params, all_param = 0, 0 for param in model.parameters(): @@ -248,10 +262,11 @@ def train(): else: batch_vision = False + transform_func = build_transform() data_module = make_supervised_data_module( tokenizer=tokenizer, data_args=data_args, - transform=model.transform, + transform=transform_func, data_collator=data_collator, slice_config=slice_config, llm_type=llm_type, diff --git a/finetune/finetune_ds.sh b/finetune/finetune_ds.sh index 5dc3a3e..92fd577 100644 --- a/finetune/finetune_ds.sh +++ b/finetune/finetune_ds.sh @@ -6,12 +6,15 @@ NODE_RANK=0 MASTER_ADDR=localhost MASTER_PORT=6001 -MODEL="openbmb/MiniCPM-Llama3-V-2_5" # or openbmb/MiniCPM-V-2 +MODEL="openbmb/MiniCPM-V-2_6" +# or openbmb/MiniCPM-V-2, openbmb/MiniCPM-Llama3-V-2_5 # ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations. # See the section for finetuning in README for more information. DATA="path/to/trainging_data" EVAL_DATA="path/to/test_data" -LLM_TYPE="llama3" # if use openbmb/MiniCPM-V-2, please set LLM_TYPE=minicpm +LLM_TYPE="qwen2" # if use openbmb/MiniCPM-V-2, please set LLM_TYPE=minicpm, if use openbmb/MiniCPM-Llama3-V-2_5, please set LLM_TYPE="llama3" + + DISTRIBUTED_ARGS=" --nproc_per_node $GPUS_PER_NODE \ @@ -28,10 +31,10 @@ torchrun $DISTRIBUTED_ARGS finetune.py \ --remove_unused_columns false \ --label_names "labels" \ --prediction_loss_only false \ - --bf16 false \ - --bf16_full_eval false \ - --fp16 true \ - --fp16_full_eval true \ + --bf16 true \ + --bf16_full_eval true \ + --fp16 false \ + --fp16_full_eval false \ --do_train \ --do_eval \ --tune_vision true \ @@ -40,8 +43,8 @@ torchrun $DISTRIBUTED_ARGS finetune.py \ --max_slice_nums 9 \ --max_steps 10000 \ --eval_steps 1000 \ - --output_dir output/output_minicpmv2 \ - --logging_dir output/output_minicpmv2 \ + --output_dir output/output_minicpmv26 \ + --logging_dir output/output_minicpmv26 \ --logging_strategy "steps" \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 1 \ diff --git a/finetune/finetune_lora.sh b/finetune/finetune_lora.sh index 96f1c09..2c12525 100644 --- a/finetune/finetune_lora.sh +++ b/finetune/finetune_lora.sh @@ -6,13 +6,14 @@ NODE_RANK=0 MASTER_ADDR=localhost MASTER_PORT=6001 -MODEL="openbmb/MiniCPM-Llama3-V-2_5" # or openbmb/MiniCPM-V-2 +MODEL="openbmb/MiniCPM-V-2_6" # or openbmb/MiniCPM-V-2, openbmb/MiniCPM-Llama3-V-2_5 # ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations. # See the section for finetuning in README for more information. DATA="path/to/trainging_data" EVAL_DATA="path/to/test_data" -LLM_TYPE="llama3" # if use openbmb/MiniCPM-V-2, please set LLM_TYPE=minicpm - +LLM_TYPE="qwen2" +# if use openbmb/MiniCPM-V-2, please set LLM_TYPE=minicpm +#if use openbmb/MiniCPM-Llama3-V-2_5, please set LLM_TYPE=llama3 DISTRIBUTED_ARGS=" --nproc_per_node $GPUS_PER_NODE \ --nnodes $NNODES \ @@ -42,12 +43,12 @@ torchrun $DISTRIBUTED_ARGS finetune.py \ --max_slice_nums 9 \ --max_steps 10000 \ --eval_steps 1000 \ - --output_dir output/output_minicpmv2_lora \ - --logging_dir output/output_minicpmv2_lora \ + --output_dir output/output__lora \ + --logging_dir output/output_lora \ --logging_strategy "steps" \ - --per_device_train_batch_size 2 \ + --per_device_train_batch_size 1 \ --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 8 \ + --gradient_accumulation_steps 1 \ --evaluation_strategy "steps" \ --save_strategy "steps" \ --save_steps 1000 \ diff --git a/finetune/readme.md b/finetune/readme.md index 4fe6026..855b583 100644 --- a/finetune/readme.md +++ b/finetune/readme.md @@ -1,6 +1,76 @@ # MiniCPM-V Finetuning +We offer the official scripts for easy finetuning of the pretrained **MiniCPM-V-2_6**, **MiniCPM-Llama3-V 2.5** and **MiniCPM-V 2.0** on downstream tasks. Our finetune scripts use transformers Trainer and DeepSpeed by default. + +### Data preparation + +To prepare your finetuning data, you should formulate each sample as a dictionary consisting of an id, an image path list with an image, and a list of conversations. Then save data samples in JSON files. + +For the vision-language example with image, you are required to provide **\<image\>** to define the position to insert the image embeddings. If you don't provide \<image\>, the image will be placed at the front of the conversation. + +<details> + <summary> + <b>vision-language example (vl_finetune_data.json) with 1 samples.</b> + </summary> + +``` + [ + { + "id": "0", + "image": 'path/to/image_0.jpg', + "conversations": [ + { + 'role': 'user', + 'content': '<image>\nHow many desserts are on the white plate?' + }, + { + 'role': 'assistant', + 'content': 'There are three desserts on the white plate.' + }, + { + 'role': 'user', + 'content': 'What type of desserts are they?' + }, + { + 'role': 'assistant', + 'content': 'The desserts are cakes with bananas and pecans on top. They share similarities with donuts, but the presence of bananas and pecans differentiates them.' + }, + { + 'role': 'user', + 'content': 'What is the setting of the image?'}, + { + 'role': 'assistant', + 'content': 'The image is set on a table top with a plate containing the three desserts.' + }, + ] + }, + ] +``` + +</details> + +### Full-parameter finetuning + +Full-parameter parameter finetuning requires updating all parameters of LLM in the whole training process. Please specify the correct MODEL path and DATA path in the shell scripts. + +```shell +MODEL="openbmb/MiniCPM-V-2_6" # or openbmb/MiniCPM-Llama3-V-2_5, openbmb/MiniCPM-V-2 +DATA="path/to/trainging_data" # json file +EVAL_DATA="path/to/test_data" # json file +``` + +To launch your training, run the following script: + +``` +sh finetune_ds.sh +``` + +#### Customizing Hyperparameters +To tailor the training process according to your specific requirements, you can adjust various hyperparameters. For comprehensive documentation on available hyperparameters and their functionalities, you can refer to the [official Transformers documentation](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments). Experimentation and fine-tuning of these parameters are essential for achieving optimal model performance tailored to your specific task and dataset. +# MiniCPM-V Finetuning + + We offer the official scripts for easy finetuning of the pretrained **MiniCPM-Llama3-V 2.5** and **MiniCPM-V 2.0** on downstream tasks. Our finetune scripts use transformers Trainer and DeepSpeed by default. ### Data preparation @@ -55,10 +125,10 @@ For the vision-language example with image, you are required to provide **\<imag Full-parameter parameter finetuning requires updating all parameters of LLM in the whole training process. Please specify the correct MODEL path, DATA path and LLM_TYPE in the shell scripts. ```shell -MODEL="openbmb/MiniCPM-Llama3-V-2_5" # or openbmb/MiniCPM-V-2 +MODEL="openbmb/MiniCPM-V-2_6" # or openbmb/MiniCPM-Llama3-V-2_5, openbmb/MiniCPM-V-2 DATA="path/to/trainging_data" # json file EVAL_DATA="path/to/test_data" # json file -LLM_TYPE="llama3" # if use openbmb/MiniCPM-V-2, please set LLM_TYPE=minicpm +LLM_TYPE="qwen2" # if use openbmb/MiniCPM-V-2, please set LLM_TYPE=minicpm, if use openbmb/MiniCPM-Llama3-V-2_5, please set LLM_TYPE="llama3" ``` To launch your training, run the following script: @@ -82,7 +152,7 @@ After training, you could load the model with the path to the adapter. We advise ``` from peft import PeftModel from transformers import AutoModel -model_type="openbmb/MiniCPM-Llama3-V-2_5" # or openbmb/MiniCPM-V-2 +model_type= "openbmb/MiniCPM-V-2_6" # or openbmb/MiniCPM-Llama3-V-2_5 , openbmb/MiniCPM-V-2 path_to_adapter="path_to_your_fine_tuned_checkpoint" model = AutoModel.from_pretrained( diff --git a/finetune/trainer.py b/finetune/trainer.py index 2bf33c4..7da95ed 100644 --- a/finetune/trainer.py +++ b/finetune/trainer.py @@ -1,3 +1,4 @@ + import torch import torch.nn as nn import deepspeed diff --git a/requirements.txt.bak b/requirements.txt.bak index df0d6ed..e274a08 100644 --- a/requirements.txt.bak +++ b/requirements.txt.bak @@ -29,5 +29,7 @@ uvicorn==0.24.0.post1 sentencepiece==0.1.99 accelerate==0.30.1 socksio==1.0.0 -gradio -gradio_client \ No newline at end of file +gradio==4.22.0 +gradio_client +http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/modelscope_studio-0.4.0.9-py3-none-any.whl +decord \ No newline at end of file diff --git a/web_demo_2.6.py b/web_demo_2.6.py new file mode 100644 index 0000000..2bebc19 --- /dev/null +++ b/web_demo_2.6.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python +# encoding: utf-8 +import torch +import argparse +from transformers import AutoModel, AutoTokenizer +import gradio as gr +from PIL import Image +from decord import VideoReader, cpu +import io +import os +import copy +import requests +import base64 +import json +import traceback +import re +import modelscope_studio as mgr + + +# README, How to run demo on different devices + +# For Nvidia GPUs. +# python web_demo_2.6.py --device cuda + +# For Mac with MPS (Apple silicon or AMD GPUs). +# PYTORCH_ENABLE_MPS_FALLBACK=1 python web_demo_2.6.py --device mps + +# Argparser +parser = argparse.ArgumentParser(description='demo') +parser.add_argument('--device', type=str, default='cuda', help='cuda or mps') +parser.add_argument('--multi-gpus', action='store_true', default=False, help='use multi-gpus') +args = parser.parse_args() +device = args.device +assert device in ['cuda', 'mps'] + +# Load model +model_path = 'openbmb/MiniCPM-V-2_6' +if 'int4' in model_path: + if device == 'mps': + print('Error: running int4 model with bitsandbytes on Mac is not supported right now.') + exit() + model = AutoModel.from_pretrained(model_path, trust_remote_code=True) +else: + if args.multi_gpus: + from accelerate import load_checkpoint_and_dispatch, init_empty_weights, infer_auto_device_map + with init_empty_weights(): + model = AutoModel.from_pretrained(model_path, trust_remote_code=True, attn_implementation='sdpa', torch_dtype=torch.bfloat16) + device_map = infer_auto_device_map(model, max_memory={0: "10GB", 1: "10GB"}, + no_split_module_classes=['SiglipVisionTransformer', 'Qwen2DecoderLayer']) + device_id = device_map["llm.model.embed_tokens"] + device_map["llm.lm_head"] = device_id # firtt and last layer should be in same device + device_map["vpm"] = device_id + device_map["resampler"] = device_id + device_id2 = device_map["llm.model.layers.26"] + device_map["llm.model.layers.8"] = device_id2 + device_map["llm.model.layers.9"] = device_id2 + device_map["llm.model.layers.10"] = device_id2 + device_map["llm.model.layers.11"] = device_id2 + device_map["llm.model.layers.12"] = device_id2 + device_map["llm.model.layers.13"] = device_id2 + device_map["llm.model.layers.14"] = device_id2 + device_map["llm.model.layers.15"] = device_id2 + device_map["llm.model.layers.16"] = device_id2 + #print(device_map) + + model = load_checkpoint_and_dispatch(model, model_path, dtype=torch.bfloat16, device_map=device_map) + else: + model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.bfloat16) + model = model.to(device=device) +tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) +model.eval() + + + + +ERROR_MSG = "Error, please retry" +model_name = 'MiniCPM-V 2.6' +MAX_NUM_FRAMES = 64 +IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.webp'} +VIDEO_EXTENSIONS = {'.mp4', '.mkv', '.mov', '.avi', '.flv', '.wmv', '.webm', '.m4v'} + +def get_file_extension(filename): + return os.path.splitext(filename)[1].lower() + +def is_image(filename): + return get_file_extension(filename) in IMAGE_EXTENSIONS + +def is_video(filename): + return get_file_extension(filename) in VIDEO_EXTENSIONS + + +form_radio = { + 'choices': ['Beam Search', 'Sampling'], + #'value': 'Beam Search', + 'value': 'Sampling', + 'interactive': True, + 'label': 'Decode Type' +} + + +def create_component(params, comp='Slider'): + if comp == 'Slider': + return gr.Slider( + minimum=params['minimum'], + maximum=params['maximum'], + value=params['value'], + step=params['step'], + interactive=params['interactive'], + label=params['label'] + ) + elif comp == 'Radio': + return gr.Radio( + choices=params['choices'], + value=params['value'], + interactive=params['interactive'], + label=params['label'] + ) + elif comp == 'Button': + return gr.Button( + value=params['value'], + interactive=True + ) + + +def create_multimodal_input(upload_image_disabled=False, upload_video_disabled=False): + return mgr.MultimodalInput(upload_image_button_props={'label': 'Upload Image', 'disabled': upload_image_disabled, 'file_count': 'multiple'}, + upload_video_button_props={'label': 'Upload Video', 'disabled': upload_video_disabled, 'file_count': 'single'}, + submit_button_props={'label': 'Submit'}) + + +def chat(img, msgs, ctx, params=None, vision_hidden_states=None): + try: + print('msgs:', msgs) + answer = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + **params + ) + res = re.sub(r'(<box>.*</box>)', '', answer) + res = res.replace('<ref>', '') + res = res.replace('</ref>', '') + res = res.replace('<box>', '') + answer = res.replace('</box>', '') + print('answer:', answer) + return 0, answer, None, None + except Exception as e: + print(e) + traceback.print_exc() + return -1, ERROR_MSG, None, None + + +def encode_image(image): + if not isinstance(image, Image.Image): + if hasattr(image, 'path'): + image = Image.open(image.path).convert("RGB") + else: + image = Image.open(image.file.path).convert("RGB") + # resize to max_size + max_size = 448*16 + if max(image.size) > max_size: + w,h = image.size + if w > h: + new_w = max_size + new_h = int(h * max_size / w) + else: + new_h = max_size + new_w = int(w * max_size / h) + image = image.resize((new_w, new_h), resample=Image.BICUBIC) + return image + ## save by BytesIO and convert to base64 + #buffered = io.BytesIO() + #image.save(buffered, format="png") + #im_b64 = base64.b64encode(buffered.getvalue()).decode() + #return {"type": "image", "pairs": im_b64} + + +def encode_video(video): + def uniform_sample(l, n): + gap = len(l) / n + idxs = [int(i * gap + gap / 2) for i in range(n)] + return [l[i] for i in idxs] + + if hasattr(video, 'path'): + vr = VideoReader(video.path, ctx=cpu(0)) + else: + vr = VideoReader(video.file.path, ctx=cpu(0)) + sample_fps = round(vr.get_avg_fps() / 1) # FPS + frame_idx = [i for i in range(0, len(vr), sample_fps)] + if len(frame_idx)>MAX_NUM_FRAMES: + frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES) + video = vr.get_batch(frame_idx).asnumpy() + video = [Image.fromarray(v.astype('uint8')) for v in video] + video = [encode_image(v) for v in video] + print('video frames:', len(video)) + return video + + +def check_mm_type(mm_file): + if hasattr(mm_file, 'path'): + path = mm_file.path + else: + path = mm_file.file.path + if is_image(path): + return "image" + if is_video(path): + return "video" + return None + + +def encode_mm_file(mm_file): + if check_mm_type(mm_file) == 'image': + return [encode_image(mm_file)] + if check_mm_type(mm_file) == 'video': + return encode_video(mm_file) + return None + +def make_text(text): + #return {"type": "text", "pairs": text} # # For remote call + return text + +def encode_message(_question): + files = _question.files + question = _question.text + pattern = r"\[mm_media\]\d+\[/mm_media\]" + matches = re.split(pattern, question) + message = [] + if len(matches) != len(files) + 1: + gr.Warning("Number of Images not match the placeholder in text, please refresh the page to restart!") + assert len(matches) == len(files) + 1 + + text = matches[0].strip() + if text: + message.append(make_text(text)) + for i in range(len(files)): + message += encode_mm_file(files[i]) + text = matches[i + 1].strip() + if text: + message.append(make_text(text)) + return message + + +def check_has_videos(_question): + images_cnt = 0 + videos_cnt = 0 + for file in _question.files: + if check_mm_type(file) == "image": + images_cnt += 1 + else: + videos_cnt += 1 + return images_cnt, videos_cnt + + +def count_video_frames(_context): + num_frames = 0 + for message in _context: + for item in message["content"]: + #if item["type"] == "image": # For remote call + if isinstance(item, Image.Image): + num_frames += 1 + return num_frames + + +def respond(_question, _chat_bot, _app_cfg, params_form): + _context = _app_cfg['ctx'].copy() + _context.append({'role': 'user', 'content': encode_message(_question)}) + + images_cnt = _app_cfg['images_cnt'] + videos_cnt = _app_cfg['videos_cnt'] + files_cnts = check_has_videos(_question) + if files_cnts[1] + videos_cnt > 1 or (files_cnts[1] + videos_cnt == 1 and files_cnts[0] + images_cnt > 0): + gr.Warning("Only supports single video file input right now!") + return _question, _chat_bot, _app_cfg + + if params_form == 'Beam Search': + params = { + 'sampling': False, + 'num_beams': 3, + 'repetition_penalty': 1.2, + "max_new_tokens": 2048 + } + else: + params = { + 'sampling': True, + 'top_p': 0.8, + 'top_k': 100, + 'temperature': 0.7, + 'repetition_penalty': 1.05, + "max_new_tokens": 2048 + } + + if files_cnts[1] + videos_cnt > 0: + params["max_inp_length"] = 4352 # 4096+256 + params["use_image_id"] = False + params["max_slice_nums"] = 1 if count_video_frames(_context) > 16 else 2 + + code, _answer, _, sts = chat("", _context, None, params) + + images_cnt += files_cnts[0] + videos_cnt += files_cnts[1] + _context.append({"role": "assistant", "content": [make_text(_answer)]}) + _chat_bot.append((_question, _answer)) + if code == 0: + _app_cfg['ctx']=_context + _app_cfg['sts']=sts + _app_cfg['images_cnt'] = images_cnt + _app_cfg['videos_cnt'] = videos_cnt + + upload_image_disabled = videos_cnt > 0 + upload_video_disabled = videos_cnt > 0 or images_cnt > 0 + return create_multimodal_input(upload_image_disabled, upload_video_disabled), _chat_bot, _app_cfg + + +def fewshot_add_demonstration(_image, _user_message, _assistant_message, _chat_bot, _app_cfg): + ctx = _app_cfg["ctx"] + message_item = [] + if _image is not None: + image = Image.open(_image).convert("RGB") + ctx.append({"role": "user", "content": [encode_image(image), make_text(_user_message)]}) + message_item.append({"text": "[mm_media]1[/mm_media]" + _user_message, "files": [_image]}) + else: + if _user_message: + ctx.append({"role": "user", "content": [make_text(_user_message)]}) + message_item.append({"text": _user_message, "files": []}) + else: + message_item.append(None) + if _assistant_message: + ctx.append({"role": "assistant", "content": [make_text(_assistant_message)]}) + message_item.append({"text": _assistant_message, "files": []}) + else: + message_item.append(None) + + _chat_bot.append(message_item) + return None, "", "", _chat_bot, _app_cfg + + +def fewshot_respond(_image, _user_message, _chat_bot, _app_cfg, params_form): + user_message_contents = [] + _context = _app_cfg["ctx"].copy() + if _image: + image = Image.open(_image).convert("RGB") + user_message_contents += [encode_image(image)] + if _user_message: + user_message_contents += [make_text(_user_message)] + if user_message_contents: + _context.append({"role": "user", "content": user_message_contents}) + + if params_form == 'Beam Search': + params = { + 'sampling': False, + 'num_beams': 3, + 'repetition_penalty': 1.2, + "max_new_tokens": 2048 + } + else: + params = { + 'sampling': True, + 'top_p': 0.8, + 'top_k': 100, + 'temperature': 0.7, + 'repetition_penalty': 1.05, + "max_new_tokens": 2048 + } + + code, _answer, _, sts = chat("", _context, None, params) + + _context.append({"role": "assistant", "content": [make_text(_answer)]}) + + if _image: + _chat_bot.append([ + {"text": "[mm_media]1[/mm_media]" + _user_message, "files": [_image]}, + {"text": _answer, "files": []} + ]) + else: + _chat_bot.append([ + {"text": _user_message, "files": [_image]}, + {"text": _answer, "files": []} + ]) + if code == 0: + _app_cfg['ctx']=_context + _app_cfg['sts']=sts + return None, '', '', _chat_bot, _app_cfg + + +def regenerate_button_clicked(_question, _image, _user_message, _assistant_message, _chat_bot, _app_cfg, params_form): + if len(_chat_bot) <= 1 or not _chat_bot[-1][1]: + gr.Warning('No question for regeneration.') + return '', _image, _user_message, _assistant_message, _chat_bot, _app_cfg + if _app_cfg["chat_type"] == "Chat": + images_cnt = _app_cfg['images_cnt'] + videos_cnt = _app_cfg['videos_cnt'] + _question = _chat_bot[-1][0] + _chat_bot = _chat_bot[:-1] + _app_cfg['ctx'] = _app_cfg['ctx'][:-2] + files_cnts = check_has_videos(_question) + images_cnt -= files_cnts[0] + videos_cnt -= files_cnts[1] + _app_cfg['images_cnt'] = images_cnt + _app_cfg['videos_cnt'] = videos_cnt + upload_image_disabled = videos_cnt > 0 + upload_video_disabled = videos_cnt > 0 or images_cnt > 0 + _question, _chat_bot, _app_cfg = respond(_question, _chat_bot, _app_cfg, params_form) + return _question, _image, _user_message, _assistant_message, _chat_bot, _app_cfg + else: + last_message = _chat_bot[-1][0] + last_image = None + last_user_message = '' + if last_message.text: + last_user_message = last_message.text + if last_message.files: + last_image = last_message.files[0].file.path + _chat_bot = _chat_bot[:-1] + _app_cfg['ctx'] = _app_cfg['ctx'][:-2] + _image, _user_message, _assistant_message, _chat_bot, _app_cfg = fewshot_respond(last_image, last_user_message, _chat_bot, _app_cfg, params_form) + return _question, _image, _user_message, _assistant_message, _chat_bot, _app_cfg + + +def flushed(): + return gr.update(interactive=True) + + +def clear(txt_message, chat_bot, app_session): + txt_message.files.clear() + txt_message.text = '' + chat_bot = copy.deepcopy(init_conversation) + app_session['sts'] = None + app_session['ctx'] = [] + app_session['images_cnt'] = 0 + app_session['videos_cnt'] = 0 + return create_multimodal_input(), chat_bot, app_session, None, '', '' + + +def select_chat_type(_tab, _app_cfg): + _app_cfg["chat_type"] = _tab + return _app_cfg + + +init_conversation = [ + [ + None, + { + # The first message of bot closes the typewriter. + "text": "You can talk to me now", + "flushing": False + } + ], +] + + +css = """ +video { height: auto !important; } +.example label { font-size: 16px;} +""" + +introduction = """ + +## Features: +1. Chat with single image +2. Chat with multiple images +3. Chat with video +4. In-context few-shot learning + +Click `How to use` tab to see examples. +""" + + +with gr.Blocks(css=css) as demo: + with gr.Tab(model_name): + with gr.Row(): + with gr.Column(scale=1, min_width=300): + gr.Markdown(value=introduction) + params_form = create_component(form_radio, comp='Radio') + regenerate = create_component({'value': 'Regenerate'}, comp='Button') + clear_button = create_component({'value': 'Clear History'}, comp='Button') + + with gr.Column(scale=3, min_width=500): + app_session = gr.State({'sts':None,'ctx':[], 'images_cnt': 0, 'videos_cnt': 0, 'chat_type': 'Chat'}) + chat_bot = mgr.Chatbot(label=f"Chat with {model_name}", value=copy.deepcopy(init_conversation), height=600, flushing=False, bubble_full_width=False) + + with gr.Tab("Chat") as chat_tab: + txt_message = create_multimodal_input() + chat_tab_label = gr.Textbox(value="Chat", interactive=False, visible=False) + + txt_message.submit( + respond, + [txt_message, chat_bot, app_session, params_form], + [txt_message, chat_bot, app_session] + ) + + with gr.Tab("Few Shot") as fewshot_tab: + fewshot_tab_label = gr.Textbox(value="Few Shot", interactive=False, visible=False) + with gr.Row(): + with gr.Column(scale=1): + image_input = gr.Image(type="filepath", sources=["upload"]) + with gr.Column(scale=3): + user_message = gr.Textbox(label="User") + assistant_message = gr.Textbox(label="Assistant") + with gr.Row(): + add_demonstration_button = gr.Button("Add Example") + generate_button = gr.Button(value="Generate", variant="primary") + add_demonstration_button.click( + fewshot_add_demonstration, + [image_input, user_message, assistant_message, chat_bot, app_session], + [image_input, user_message, assistant_message, chat_bot, app_session] + ) + generate_button.click( + fewshot_respond, + [image_input, user_message, chat_bot, app_session, params_form], + [image_input, user_message, assistant_message, chat_bot, app_session] + ) + + chat_tab.select( + select_chat_type, + [chat_tab_label, app_session], + [app_session] + ) + chat_tab.select( # do clear + clear, + [txt_message, chat_bot, app_session], + [txt_message, chat_bot, app_session, image_input, user_message, assistant_message] + ) + fewshot_tab.select( + select_chat_type, + [fewshot_tab_label, app_session], + [app_session] + ) + fewshot_tab.select( # do clear + clear, + [txt_message, chat_bot, app_session], + [txt_message, chat_bot, app_session, image_input, user_message, assistant_message] + ) + chat_bot.flushed( + flushed, + outputs=[txt_message] + ) + regenerate.click( + regenerate_button_clicked, + [txt_message, image_input, user_message, assistant_message, chat_bot, app_session, params_form], + [txt_message, image_input, user_message, assistant_message, chat_bot, app_session] + ) + clear_button.click( + clear, + [txt_message, chat_bot, app_session], + [txt_message, chat_bot, app_session, image_input, user_message, assistant_message] + ) + + with gr.Tab("How to use"): + with gr.Column(): + with gr.Row(): + image_example = gr.Image(value="http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/m_bear2.gif", label='1. Chat with single or multiple images', interactive=False, width=400, elem_classes="example") + example2 = gr.Image(value="http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/video2.gif", label='2. Chat with video', interactive=False, width=400, elem_classes="example") + example3 = gr.Image(value="http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/fshot.gif", label='3. Few shot', interactive=False, width=400, elem_classes="example") + + +# launch +demo.launch(share=False, debug=True, show_api=False, server_port=8885, server_name="0.0.0.0") +