
## Installation

```bash
pip install transformers -e .
apt update && apt install -y \
    build-essential \
    python3-dev \
    pkg-config \
    libcairo2-dev \
    python3-cairo \
    libgirepository1.0-dev \
    && apt clean && rm -rf /var/lib/apt/lists/*
pip install --no-cache-dir \
    -r requirements.txt \
    --extra-index-url https://download.pytorch.org/whl/cu121
pip install pycairo>=1.16.0
```

**Evaluation of Qwen2.5-VL on Mantis**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-3B-Instruct,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 8
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 1.154
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-32B-Instruct,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 1.04
```

**Evaluation of InternVL3 on Mantis**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=OpenGVLab/InternVL3-1B-hf, device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 2.5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-2B-hf,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 9.5
```

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-8B-hf,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.06
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-14B-hf,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 1.29
```

**Evaluation of LLaVA-OneVision on Mantis**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-0.5b-ov-hf,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0 \
    --vision_start True \
    --vision_end True \
    --scale 1.5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-7b-ov-hf,device_map=cuda \
    --tasks mantis \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 9
```



**Evaluation of Qwen2.5-VL on Muirbench**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-3B-Instruct,device_map=cuda \
    --tasks muirbench \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,device_map=cuda \
    --tasks muirbench \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-32B-Instruct,device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 1.03
```

**Evaluation of InternVL3 on Muirbench**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=OpenGVLab/InternVL3-1B-hf, device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 10
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-2B-hf,device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 5.421
```

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-8B-hf,device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.01
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-14B-hf,device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 1.1
```

**Evaluation of LLaVA-OneVision on Muirbench**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-0.5b-ov-hf,device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0 \
    --vision_start True \
    --vision_end True \
    --scale 4.3
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-7b-ov-hf,device_map=cuda \
    --tasks muirbench  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 3.8
```

**Evaluation of Qwen2.5-VL on Mirb**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-3B-Instruct,device_map=cuda \
    --tasks mirb \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 6.3
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,device_map=cuda \
    --tasks mirb \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 1.058
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-32B-Instruct,device_map=cuda \
    --tasks mirb  \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 1.174
```

**Evaluation of InternVL3 on Mirb**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=OpenGVLab/InternVL3-1B-hf,device_map=cuda \
    --tasks mirb \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-2B-hf,device_map=cuda \
    --tasks mirb  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.16
```

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-8B-hf,device_map=cuda \
    --tasks mirb  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.021
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-14B-hf,device_map=cuda \
    --tasks mirb  \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 2
```

**Evaluation of LLaVA-OneVision on MiRB**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-0.5b-ov-hf,device_map=cuda \
    --tasks mirb  \
    --batch_size 1 \
    --select_layer 0 \
    --vision_start True \
    --vision_end True \
    --scale 1.5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-7b-ov-hf,device_map=cuda \
    --tasks mirb  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.3
```

**Evaluation of Qwen2.5-VL on Qbench2**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-3B-Instruct,device_map=cuda \
    --tasks qbench2_dev \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 1.5
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,device_map=cuda \
    --tasks qbench2_dev \
    --batch_size 1 \
    --select_layer 0,1,2,3 \
    --vision_start True \
    --vision_end True \
    --scale 1.2
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=Qwen/Qwen2.5-VL-32B-Instruct,device_map=cuda \
    --tasks qbench2_dev  \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 1.03
```

**Evaluation of InternVL3 on Qbench2**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model qwen2_5_vl \
    --model_args pretrained=OpenGVLab/InternVL3-1B-hf,device_map=cuda \
    --tasks qbench2_dev \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.209
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-2B-hf,device_map=cuda \
    --tasks qbench2_dev  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 3.28
```

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-8B-hf,device_map=cuda \
    --tasks qbench2_dev  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.0009
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model internvl3 \
    --model_args pretrained=OpenGVLab/InternVL3-14B-hf,device_map=cuda \
    --tasks qbench2_dev  \
    --batch_size 1 \
    --select_layer 0,1 \
    --vision_start True \
    --vision_end True \
    --scale 6.6
```

**Evaluation of LLaVA-OneVision on Qbench2**

```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-0.5b-ov-hf,device_map=cuda \
    --tasks qbench2_dev  \
    --batch_size 1 \
    --select_layer 0 \
    --vision_start True \
    --vision_end True \
    --scale 1.06
```
```bash
CUDA_VISIBLE_DEVICES=0 accelerate launch --num_processes 1 --main_process_port 12345 -m lmms_eval \
    --model llava_onevision \
    --model_args pretrained=llava-hf/llava-onevision-qwen2-7b-ov-hf,device_map=cuda \
    --tasks qbench2_dev  \
    --batch_size 1 \
    --select_layer 0,1,2 \
    --vision_start True \
    --vision_end True \
    --scale 1.45
```