From 18f3812c4f9af2e49af6b50e8afe7f607b8a75d6 Mon Sep 17 00:00:00 2001 From: Pu Fanyi Date: Wed, 10 Jul 2024 18:13:43 -0700 Subject: [PATCH 01/36] Load tasks only one time (#139) * chore: Initialize tasks only once to avoid re-initialization * chore: Initialize tasks only once to avoid re-initialization * chore: Refactor task initialization to avoid re-initialization * chore: Update task initialization to fix include_path issue * chore: Update task initialization to fix include_path issue --- lmms_eval/api/registry.py | 1 + lmms_eval/tasks/__init__.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/lmms_eval/api/registry.py b/lmms_eval/api/registry.py index 6b911997a..556a95282 100755 --- a/lmms_eval/api/registry.py +++ b/lmms_eval/api/registry.py @@ -33,6 +33,7 @@ def get_model(model_name): TASK_REGISTRY = {} # Key: task name, Value: task ConfigurableTask class GROUP_REGISTRY = {} # Key: group name, Value: list of task names or group names +TASK_INITIALIZED = False ALL_TASKS = set() # Set of all task names and group names func2task_index = {} # Key: task ConfigurableTask class, Value: task name diff --git a/lmms_eval/tasks/__init__.py b/lmms_eval/tasks/__init__.py index fadbb7bff..ce11b8524 100755 --- a/lmms_eval/tasks/__init__.py +++ b/lmms_eval/tasks/__init__.py @@ -11,6 +11,7 @@ TASK_REGISTRY, GROUP_REGISTRY, ALL_TASKS, + TASK_INITIALIZED, ) from loguru import logger @@ -111,9 +112,14 @@ def include_path(task_dir): def initialize_tasks(verbosity="INFO"): logger.remove() eval_logger.add(sys.stdout, colorize=True, level=verbosity) + global TASK_INITIALIZED + if TASK_INITIALIZED: + eval_logger.info("Tasks already initialized, skipping re-initialization.") + return # eval_logger.add(sys.stderr, level=verbosity) task_dir = os.path.dirname(os.path.abspath(__file__)) + "/" include_path(task_dir) + TASK_INITIALIZED = True def get_task(task_name, model_name): From 4f8db1d37b1f824432927e74d6d82e06bb5aaed1 Mon Sep 17 00:00:00 2001 From: Pu Fanyi Date: Fri, 12 Jul 2024 17:26:50 -0700 Subject: [PATCH 02/36] Upload live_bench results (#140) * upload results * add a readme * chore: Update upload_results.py script to use shell syntax * Update upload_results.py * Update upload_results.py --- tools/live_bench/script/README.md | 15 ++ tools/live_bench/script/upload_results.py | 201 ++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 tools/live_bench/script/README.md create mode 100644 tools/live_bench/script/upload_results.py diff --git a/tools/live_bench/script/README.md b/tools/live_bench/script/README.md new file mode 100644 index 000000000..49250cfc5 --- /dev/null +++ b/tools/live_bench/script/README.md @@ -0,0 +1,15 @@ +## Usage + +### Upload Results + +```sh +python upload_results.py -f -m [-F] +``` + +`[-F]` means the script will automatically upload the results without human checking. Otherwise, the script will print the results and ask for confirmation before uploading. + +Example: + +```sh +python upload_results.py -f logs/0706_0959_model_outputs_gpt4v_model_args_c974bc -m gpt-4o -F +``` diff --git a/tools/live_bench/script/upload_results.py b/tools/live_bench/script/upload_results.py new file mode 100644 index 000000000..9b7e45bf8 --- /dev/null +++ b/tools/live_bench/script/upload_results.py @@ -0,0 +1,201 @@ +import os +import json +import argparse +import datasets +import numpy as np +import pandas as pd +from datasets import Dataset, load_dataset + +from PIL import Image +from typing import Dict, List, Union +from tqdm import tqdm + + +EPS = 1e-6 + +RESULT_FEATURES = { + "id": datasets.Value("int32"), + "images": datasets.Sequence(datasets.Image()), + "question": datasets.Value("string"), + "ground_truth": datasets.Value("string"), + "criteria": datasets.Value("string"), + "subtask": datasets.Value("string"), + "response": datasets.Value("string"), + "score": datasets.Value("int32"), + "reason": datasets.Value("string"), +} + +SUBTASKS = [ + "Basic Understanding", + "Contextual Analysis", + "Deeper Implications", + "Broader Implications", + "Further Insights", +] + + +def load_images(config) -> Dict[int, List[Image.Image]]: + dataset = datasets.load_dataset( + config["dataset_path"], config["dataset_name"], split=config["test_split"] + ) + images = {} + for data in tqdm(dataset, desc="Loading images"): + images[data["id"]] = data["images"] + return images + + +def get_hf_results(results, detailed_results): + live_bench_images = load_images(results["configs"]["live_bench"]) + mapping = {k: [] for k in RESULT_FEATURES.keys()} + for result in tqdm(detailed_results["logs"], desc="Loading results"): + doc = result["doc"] + res = {} + res["id"] = doc["id"] + res["images"] = live_bench_images[doc["id"]] + res["question"] = doc["question"] + res["ground_truth"] = doc["answer"] + res["criteria"] = doc["criteria"] + res["subtask"] = doc["subtask"] + res["response"] = result["filtered_resps"][0] + res["score"] = result["gpt4_eval_score"]["rating"] + res["reason"] = result["gpt4_eval_score"]["explanation"] + for k, v in res.items(): + mapping[k].append(v) + result_dataset = datasets.Dataset.from_dict( + mapping=mapping, features=datasets.Features(RESULT_FEATURES) + ) + return result_dataset + + +def preview_results(results, heading: str): + HEADING = "=" * 15 + " " + heading + " " + "=" * 15 + ENDING = "=" * len(HEADING) + print(HEADING) + print(results) + print(ENDING) + + +def calculate_score(results: Dataset): + results = results.to_pandas() + + sum_score, count = 0, 0 + score = {} + for subtask in SUBTASKS: + score[subtask] = [] + for index, result in tqdm( + results.iterrows(), total=len(results), desc="Calculating score" + ): + if result["score"] == -1: + continue + sum_score += result["score"] / 10 + count += 1 + subtask = result["subtask"] + if subtask not in SUBTASKS: + subtask = "Further Insights" + score[result["subtask"]].append(result["score"] / 10) + res = [ + (subtask, len(score[subtask]), np.mean(score[subtask]) * 100) + for subtask in SUBTASKS + ] + res.append(("Total", count, sum_score / count * 100)) + res = pd.DataFrame(res, columns=["Subtask", "Count", "Score"]) + + return res + + +def get_results(folder): + detailed_file = os.path.join(folder, "live_bench.json") + results_file = os.path.join(folder, "results.json") + + with open(results_file, "r") as f: + results = json.load(f) + + assert ( + "live_bench" in results["configs"] + ), "No live_bench config found in results.json" + final_score = results["results"]["live_bench"]["gpt4_eval_score,none"] + model_configs = results["model_configs"] + version = results["configs"]["live_bench"]["metadata"]["version"] + + assert ( + model_configs["limit"] is None + ), "Model limit is not None, please check if the model is tested on the full dataset" + + with open(detailed_file, "r") as f: + detailed_results = json.load(f) + + hf_results = get_hf_results(results, detailed_results) + preview_results(hf_results.to_pandas().iloc[0], "Detailed Results") + score = calculate_score(hf_results) + preview_results(score, "Final Score") + + assert ( + abs(score[score["Subtask"] == "Total"]["Score"] - final_score) <= EPS + ).all(), "Final score does not match the calculated score" + + return hf_results, score, version + + +def upload_results( + hf_results: Dataset, + score: pd.DataFrame, + model_name, + dataset_version, + log_folder="logs", +): + hf_results.push_to_hub( + "lmms-lab/LiveBenchDetailedResults", + config_name=dataset_version, + split=model_name.replace("-", "_"), + ) + if not os.path.exists(log_folder): + os.makedirs(log_folder) + score_path = os.path.abspath( + os.path.join(log_folder, f"{dataset_version}_{model_name}.csv") + ) + score.to_csv(score_path, index=False) + print(f"Results saved to {score_path}") + score_dict = {item["Subtask"]: item["Score"] for index, item in score.iterrows()} + score_dict["Model Name"] = model_name + try: + hf_score = datasets.load_dataset( + "lmms-lab/LiveBenchResults", dataset_version, split="test" + ) + except: + hf_score = Dataset.from_dict( + {subtask: [] for subtask in ["Model Name", "Total"] + SUBTASKS} + ) + hf_score = hf_score.add_item(score_dict) + df_score = pd.DataFrame(hf_score) + df_score = df_score.drop_duplicates(subset=["Model Name"], keep="last") + df_score = df_score[["Model Name", "Total"] + SUBTASKS] + hf_score = Dataset.from_pandas(df_score) + hf_score.push_to_hub("lmms-lab/LiveBenchResults", dataset_version, split="test") + + +if __name__ == "__main__": + argparse = argparse.ArgumentParser() + argparse.add_argument( + "--folder", "-f", type=str, required=True, help="Results folder" + ) + argparse.add_argument("--name", "-m", type=str, required=True, help="Model name") + argparse.add_argument( + "--log_folder", "-l", type=str, default="logs", help="Log folder" + ) + argparse.add_argument("--force", "-F", action="store_true", help="Force upload") + args = argparse.parse_args() + hf_results, score, version = get_results(args.folder) + print( + f"Results will be uploaded with model name {args.name} and model version {version}" + ) + if args.force is False: + print("Are you sure you want to upload the results? (y/n)", end=" ") + while True: + choice = input().lower() + if choice == "y": + break + elif choice == "n": + exit() + else: + print("Invalid choice, please enter 'y' or 'n'") + upload_results(hf_results, score, args.name, version, args.log_folder) From 5fc5f2f5acf454fc99448b0d62eb52b4bffba0d5 Mon Sep 17 00:00:00 2001 From: Kaichen Zhang - NTU Date: Tue, 16 Jul 2024 10:12:11 +0800 Subject: [PATCH 03/36] Add Muirbench (#143) * handle gen kwargs in internvl2 * Add muirbench --- lmms_eval/models/internvl2.py | 8 ++ lmms_eval/tasks/muirbench/muirbench.yaml | 41 ++++++++ lmms_eval/tasks/muirbench/utils.py | 117 +++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 lmms_eval/tasks/muirbench/muirbench.yaml create mode 100644 lmms_eval/tasks/muirbench/utils.py diff --git a/lmms_eval/models/internvl2.py b/lmms_eval/models/internvl2.py index 5f4365d01..730493d9e 100644 --- a/lmms_eval/models/internvl2.py +++ b/lmms_eval/models/internvl2.py @@ -209,6 +209,14 @@ def generate_until(self, requests) -> List[str]: if k not in gen_kwargs: gen_kwargs[k] = v + pop_keys = [] + for k, v in gen_kwargs.items(): + if k not in DEFAULT_GEN_KWARGS: + pop_keys.append(k) + + for k in pop_keys: + gen_kwargs.pop(k) + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] visuals = self.flatten(visuals) if self.modality == "image": diff --git a/lmms_eval/tasks/muirbench/muirbench.yaml b/lmms_eval/tasks/muirbench/muirbench.yaml new file mode 100644 index 000000000..43b8ab7cc --- /dev/null +++ b/lmms_eval/tasks/muirbench/muirbench.yaml @@ -0,0 +1,41 @@ + +dataset_path: MUIRBENCH/MUIRBENCH +task: "muirbench" +dataset_kwargs: + token: True +test_split: test +output_type: generate_until +doc_to_visual: !function utils.muir_doc_to_visual +doc_to_text: !function utils.muir_doc_to_text +doc_to_target: !function utils.muir_doc_to_target +process_results: !function utils.muir_process_results + +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nAnswer with the option's letter from the given choices directly." + + +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + do_sample: False + +filter_list: + - name: "flexible-extract" + filter: + - function: !function utils.MultiChoiceRegexFilter + group_select: 0 + ignore_case: true + ignore_punctuation: true + regex_pattern: "([A-Z])\\." + +metric_list: + - metric: muirbench_score_overall + aggregation: !function utils.muir_aggregation + higher_is_better: true + ignore_case: true + ignore_punctuation: true + +metadata: + - version: 0.0 diff --git a/lmms_eval/tasks/muirbench/utils.py b/lmms_eval/tasks/muirbench/utils.py new file mode 100644 index 000000000..2924edac4 --- /dev/null +++ b/lmms_eval/tasks/muirbench/utils.py @@ -0,0 +1,117 @@ + +from lmms_eval.filters.extraction import ExtendedRegexFilter +from lmms_eval.filters.transformation import MapFilter +import re +import pandas as pd + + +def muir_doc_to_text(doc, model_specific_prompt_kwargs=None): + question, choices = doc["question"], doc["options"] + len_choices = len(choices) + post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = model_specific_prompt_kwargs["pre_prompt"] + options = [chr(ord("A") + i) for i in range(len_choices)] + choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) + return f"{pre_prompt}{question}\n{choices_str}{post_prompt}" + + +def muir_doc_to_visual(doc): + image_list = [image.convert("RGB") for image in doc["image_list"]] + return image_list + + +def muir_doc_to_target(doc): + return doc["answer"] + + +def muir_process_results(doc, result): + pred = result[0] + task = doc["task"] + idx = doc["idx"] + image_relation = doc["image_relation"] + answer = doc["answer"] + image_type = doc["image_type"] + + data_dict = { + "pred" : pred, + "task" : task, + "idx" : idx, + "image_relation" : image_relation, + "answer" : answer, + "image_type" : image_type, + } + + return {"muirbench_score_overall" : data_dict} + + +def muir_aggregation(results): + task_num = {} + score = 0 + task_score = {} + for result in results: + if result["task"] not in task_score: + task_score[result["task"]] = 0 + + if result["task"] not in task_num: + task_num[result["task"]] = 0 + + if result["pred"].lower().strip() == result["answer"].lower().strip(): + task_score[result["task"]] += 1 + score += 1 + task_num[result["task"]] += 1 + + score = score / len(results) + + task_score = {k : v / task_num[k] for k,v in task_score.items()} + + print("=" * 50) + for k, v in task_score.items(): + print(f"{k} : {v:.2f}") + print("=" * 50) + + return score + + + + +class MultiChoiceRegexFilter(ExtendedRegexFilter): + def __init__(self, *args, **kwargs): + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(*args, **kwargs) + + def apply(self, resps, docs): + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + filtered_resps = [] + + for r, doc in zip(resps, docs): + # Regex to directly extract the option letter from the model response + option_letter_regex = re.compile(r"^\s*([A-Z])\.") + + # Process each response + filtered = [] + for resp in r: + # Try to match the option letter at the start of the response + match = option_letter_regex.match(resp) + if match: + # If a match is found, append the matched letter + filtered.append(match.group(1)) + else: + # If no match, return the original response + filtered.append(resp) + + # Assuming we need the first response that matches or the original response + filtered_resps.append(filtered[0]) + + return filtered_resps From b2a009b6bbf8353172f5a1dd9c29ea1f67610c02 Mon Sep 17 00:00:00 2001 From: Pu Fanyi Date: Mon, 15 Jul 2024 19:12:25 -0700 Subject: [PATCH 04/36] if no response directly return 0 (#142) --- lmms_eval/tasks/live_bench/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lmms_eval/tasks/live_bench/utils.py b/lmms_eval/tasks/live_bench/utils.py index f83812936..ae0b1ac4c 100644 --- a/lmms_eval/tasks/live_bench/utils.py +++ b/lmms_eval/tasks/live_bench/utils.py @@ -165,8 +165,8 @@ def livebench_process_results(doc, results): criteria = doc["criteria"] if subtask not in SUBTASKS: subtask = "further insights" - if not results: - return {"gpt4_eval_score": {"rating": -1, "explanation": "No response", "model_name": "N/A", "subtask": subtask}} + if not results or results[0] == "": + return {"gpt4_eval_score": {"rating": 0, "explanation": "No response", "model_name": "N/A", "subtask": subtask}} rating, explanation, model_name = get_chat_response(base64_images=base64_images, question=doc["question"], ground_truth_answer=doc["answer"], answer=results[0] if results else "", criteria=criteria) if rating >= 0: return {"gpt4_eval_score": {"rating": rating, "explanation": explanation, "model_name": model_name, "subtask": subtask, "id": doc["id"]}} From dfdba507b5fbe985b0030ffec575f9f2638bc1ed Mon Sep 17 00:00:00 2001 From: Li Bo Date: Tue, 16 Jul 2024 11:13:52 +0800 Subject: [PATCH 05/36] merge ov evals (#144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update gpt_eval_model_name to "gpt-3.5-turbo" in mathvista.yaml * Squashed commit of the following: commit 994c9f97a2f8db3e9b7d7933d1e1680acde5b70b Author: Yan Shu <570533048@qq.com> Date: Mon Jul 8 17:21:23 2024 +0800 Add files via upload * Squashed commit of the following: commit e31cd7883d4555c7530795c7f102b8d78cbd372f Author: Bo Li Date: Wed Jul 10 12:08:08 2024 +1000 chore: Update lmms_eval/models/vila.py and lmms_eval/tasks/__init__.py commit 1d8c980d1089f9d7702c3b92d5c85039f2809c6d Author: kcz358 Date: Tue Jul 9 02:08:52 2024 +0000 Rename xcomposer 4KHD commit 6da76f36ecf5f9aa73057e767a4fcb60c99ff896 Author: Bo Li Date: Tue Jul 9 11:55:56 2024 +1000 Upgrade lmms-eval to version 0.2.1 commit cd1858523fcd8630082cbefba8710e0de3ee8805 Author: Bo Li Date: Tue Jul 9 11:52:23 2024 +1000 Upgrade lmms-eval to support more models and evaluation tasks commit 672d7e5bb49dcb34e1b2fdeb09f3f4588dc583a6 Author: Bo Li Date: Tue Jul 9 11:43:41 2024 +1000 feat: Add tie_weights parameter to Llava model initialization commit 2037a86261b55fa42b8ba3a04eab192b3e69d6ea Merge: e6844db1 a5c18692 Author: Bo Li Date: Tue Jul 9 11:37:12 2024 +1000 Fix gen kwargs image aspect ratio in internvl2 commit a5c186925de989b616f58a35ece36065a32b4594 Merge: 2ebec77f 557083a1 Author: Li Bo Date: Tue Jul 9 09:15:56 2024 +0800 Merge pull request #137 from shuyansy/main add MLVU task commit 557083a156c3dd67ac79e22b4202e9b69b6b00f4 Author: Yan Shu <570533048@qq.com> Date: Mon Jul 8 16:56:50 2024 +0800 Add files via upload commit 2ebec77f5606d79e9a7b995970e32792050606a1 Merge: 211bfede b23d349e Author: Li Bo Date: Mon Jul 8 11:53:06 2024 +0800 Merge pull request #136 from Dousia/main Add detailcaps commit b23d349e46d60dc149ffaa54d6e019f4996ed92d Author: ByteDance Date: Sun Jul 7 23:24:19 2024 +0800 Add install capture_metric in env commit c6e211d5f9dbb7572d3a141b6504cb1ca2007c33 Author: ByteDance Date: Sun Jul 7 23:04:13 2024 +0800 Add detailcaps commit 211bfedebad243ef82a8b0be36c3b5a9b9cb2f72 Merge: 7c208b76 79514eee Author: Li Bo Date: Tue Jul 2 23:05:12 2024 +0800 Merge pull request #133 from EvolvingLMMs-Lab/dev/wild_vision Add wild vision bench commit 79514eeebcfd6f655be2a10c776037d12a7b7214 Author: kcz358 Date: Mon Jul 1 15:10:02 2024 +0000 Fixing handling None filtered score commit 725fac2781446958b905e1e6c6eb3c0a8e582e49 Author: kcz358 Date: Mon Jul 1 08:25:42 2024 +0000 Fixing dataset name commit 8d963e132ac03fc0d835d480cfcfcabe72af143c Author: kcz358 Date: Mon Jul 1 08:24:51 2024 +0000 Fixing scoring logic commit e2990d0a69e876721256fdf946c68ba7ae0cbdc1 Author: kcz358 Date: Mon Jul 1 06:06:57 2024 +0000 Hardcode to keep image for wild vision commit ed381736730d8fb785b4ee919fdb751734ecef25 Author: kcz358 Date: Mon Jul 1 06:06:38 2024 +0000 Add wild vision 0617 commit 7c208b76640c986cfe94233dce735c3ca4ad4319 Author: Li Bo Date: Mon Jul 1 11:53:31 2024 +0800 Update README.md commit 39d40dea47bc59ff04e8b0cbc445345098debc9a Merge: e19b43a3 ba7081c0 Author: Li Bo Date: Mon Jul 1 11:47:09 2024 +0800 Merge pull request #129 from Dannoopsy/mmbench_ru add task MMBench-ru commit e19b43a3a1e7212e623061b164b0419cc0dda689 Merge: 11fd7e3f a0de8970 Author: Li Bo Date: Mon Jul 1 11:46:58 2024 +0800 Merge pull request #128 from Dannoopsy/gqa-ru add task gqa-ru commit 11fd7e3fc05908aeb01e4a6161a7b55cd38b3122 Merge: 383e7fea a7522592 Author: Li Bo Date: Mon Jul 1 11:46:16 2024 +0800 Merge pull request #130 from lscpku/vitatecs Add task VITATECS commit a75225926e5954f85466d257f99acf0163fde596 Author: lscpku Date: Fri Jun 28 20:37:06 2024 +0800 create new task vitatecs commit ba7081c0abac840002d320e30733e891298dfa11 Author: Dannoopsy <63581325+Dannoopsy@users.noreply.github.com> Date: Fri Jun 28 12:21:05 2024 +0300 change prompt to ru commit 27ea9c0055a8abf3a8198829b8617018479918e2 Author: Dannoopsy Date: Thu Jun 27 17:17:29 2024 +0000 add mmbench_ru_dev commit 383e7fead3138aedf62e9c0ec48303835ef26e2a Merge: 06fa000f ed2e7f79 Author: Li Bo Date: Fri Jun 28 00:14:10 2024 +0800 Merge pull request #126 from lorenzomammana/feature/external-package-integration External package integration using plugins commit ed2e7f792151d21bce8f1c498270b9391e1d5c85 Merge: 03947e14 06fa000f Author: Lorenzo Mammana Date: Thu Jun 27 15:38:10 2024 +0000 Merge branch 'main' into feature/external-package-integration commit a0de89708d5e6f259bb17f0eaace3c5b901b275c Author: Dannoopsy Date: Tue Jun 25 11:11:37 2024 +0000 new task gqa-ru commit 06fa000f60d3e4d160fac8ceb9959ae92a98f752 Author: kcz358 Date: Tue Jun 25 06:41:13 2024 +0000 Fix vid mme post prompt issue commit b388d79e0df6f60068196cb7047453ebd22d6ef1 Author: Li Bo Date: Sun Jun 23 22:31:16 2024 +0800 Update activitynetqa_generation.yaml commit 8f9d620fcd9d0a0742ee6bcf51ea63bd6b088a36 Author: Li Bo Date: Sun Jun 23 14:02:25 2024 +0800 Update pyproject.toml commit 6341b7c15ce9fb28eb06b067ddb299d6cf2e16c3 Merge: fce85f1b 903b042b Author: Li Bo Date: Sun Jun 23 14:02:02 2024 +0800 Merge pull request #125 from EvolvingLMMs-Lab/dev/interleave [Model] aligned llava-interleave model results on video tasks commit 903b042be016016d4ebeecb07701f3076a2d323c Author: kcz358 Date: Sat Jun 22 12:07:13 2024 +0000 Remove unnecessary lines for video llava commit d78ec86407b729a964906a8c2e50704b4bc74d06 Merge: ebe7217a fce85f1b Author: Li Bo Date: Sat Jun 22 13:57:31 2024 +0800 Merge branch 'main' into dev/interleave commit ebe7217a486c1e754e42c2cbdb834e09fbbcc9b0 Author: kcz358 Date: Sat Jun 22 02:57:08 2024 +0000 Delete unnecessary lines commit 120c474b056f9177c74e1fd9691d59e2f234b785 Author: kcz358 Date: Fri Jun 21 08:38:41 2024 +0000 Revise model registry for llava_hf and longva commit 7d6201f921088afd3f52a35076e3c6fcc9aa518c Author: kcz358 Date: Fri Jun 21 08:38:24 2024 +0000 Add longva commit 12f480699c71a12a24d4349d9b0681933201a3a6 Author: kcz358 Date: Fri Jun 21 08:35:39 2024 +0000 Remove unnecessary lines since use batched visuals now in llava commit 12cea76f1f0f14b1fd1007c9d39a9b0557368637 Author: Bo Li Date: Thu Jun 20 18:15:32 2024 +0000 chore: Add loguru for logging in lmms_eval package commit 03947e14a46fd25b412931f7c9c25f4a2971d0b4 Author: Lorenzo Mammana Date: Wed Jun 5 13:40:41 2024 +0000 feat: Allow including external tasks from plugins commit b80a91f73e15ddd0b0ce1322d7d121fa14030eed Author: Lorenzo Mammana Date: Wed Jun 5 13:04:55 2024 +0000 feat: Allow loading model configurations from other packages commit 8ef24740dd48a11c97eb627f2fff4aca107fef0d Author: Bo Li Date: Thu Jun 20 12:11:03 2024 +0000 chore: Remove unused models from lmms_eval package commit af38885fc2e066f5ea44388f33e07176f836fe28 Author: Bo Li Date: Thu Jun 20 12:07:09 2024 +0000 chore: Handle ImportError when importing models Handle the ImportError exception when importing models in the lmms_eval package. This change adds a try-except block to catch the ImportError and print an error message indicating the failed import. This will help with troubleshooting and identifying any issues with the model imports. commit fce85f1b03ff7043b29dee787c5d17a08dd2687a Merge: dbe63293 d94f83cb Author: Li Bo Date: Thu Jun 20 20:02:12 2024 +0800 Merge pull request #120 from EvolvingLMMs-Lab/pufanyi/hf_dataset_docs Add docs for datasets upload to HF commit dbe63293245a5141fdfd80bda7657c304f6bd32f Author: choiszt Date: Thu Jun 20 15:14:21 2024 +0800 update ablation for videomme datasets commit d94f83cb3f08b61a2c75cc4326e58792100605b3 Author: Li Bo Date: Thu Jun 20 13:30:59 2024 +0800 Update README.md commit cab8159ff35db330536c0b6dfb4b0a3b24142209 Author: Li Bo Date: Thu Jun 20 13:30:29 2024 +0800 Update README.md commit 45876652a877a8006b828f32f5cc4660629f9190 Author: kcz358 Date: Thu Jun 20 03:55:30 2024 +0000 Add llava_hf back to registry commit 3463651b8c54d36cd94169e3d376f5ed225a195a Author: kcz358 Date: Thu Jun 20 03:54:33 2024 +0000 Remove handling non-visual loop in llava commit cb0d3f49b72790b081f981e0e6147131542f7f68 Author: Fanyi Pu Date: Thu Jun 20 02:11:18 2024 +0800 update readme commit 813877bfe5ac590cdbe92dd74d18f83a2091f748 Author: Fanyi Pu Date: Wed Jun 19 15:37:52 2024 +0800 to sh script commit a14684b8557d5894976448a5c559ed7a66a6cf16 Author: Fanyi Pu Date: Wed Jun 19 15:37:04 2024 +0800 lint commit d0f8851d42ba31f5da2a7a65e91499db45174dbc Author: Fanyi Pu Date: Wed Jun 19 15:36:48 2024 +0800 small fix commit 63748e9718f287ad433afc90e340b5e17a89c1ed Author: Fanyi Pu Date: Wed Jun 19 15:36:43 2024 +0800 small fix commit 7f1159a1fe04cfb783dc31d4fbdef3bda0ce19e4 Author: Fanyi Pu Date: Wed Jun 19 15:35:05 2024 +0800 update preparation commit 19f9bd621c76a483ff98f8c7eb78f64753da683a Author: Fanyi Pu Date: Wed Jun 19 15:23:24 2024 +0800 docs commit ce6f889ba02d819979c7922f6336cf4f1f718f65 Author: Fanyi Pu Date: Wed Jun 19 15:04:16 2024 +0800 tutorial commit f513c520c2a3dad26d2b2ca5c4ed4db05a493c73 Author: Bo Li Date: Wed Jun 19 06:51:19 2024 +0000 chore: Update dependencies to fix potential risks and improve compatibility commit efb529552c5e4ba039a4cba8e9aa5cb7ba65bf90 Author: kcz358 Date: Wed Jun 19 10:25:58 2024 +0800 Release llava-wilder commit 742651fc9daf97e2f57831ed6e6e7ee7ead7d555 Author: Fanyi Pu Date: Wed Jun 19 07:44:26 2024 +0800 feat: Add support for auto downloading tar format videos commit 511b6259828212fcba954cdeb8cf90d6e5daabf8 Merge: 22a4958e 050b2c37 Author: Bo Li Date: Tue Jun 18 17:01:03 2024 +0000 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval commit 050b2c370017e9b97475dd6cf01fd051b5ca5c86 Merge: 74facb41 ef306512 Author: Li Bo Date: Tue Jun 18 13:13:38 2024 +0800 Merge pull request #114 from zjysteven/add-tinyllava add tinyllava commit ef306512e5135f76dffa383f600b8733015836e8 Author: Jingyang Zhang Date: Mon Jun 17 17:57:02 2024 -0400 fix typo commit 9bab67732a4238097725deddf867fb1946ffee40 Merge: dbfb2387 74facb41 Author: Jingyang Zhang Date: Sun Jun 16 10:56:05 2024 -0400 Merge branch 'EvolvingLMMs-Lab:main' into add-tinyllava commit 74facb41a826691dfce4458cf1d8659b34fc5bf5 Merge: 8ba192f9 d5df72de Author: Li Bo Date: Sun Jun 16 17:59:19 2024 +0800 Merge pull request #118 from teowu/main Fix the potential risk by PR #117 commit d5df72de2d03108d6b365818ecc3551ac9aa6302 Merge: 5bf59ed2 8ba192f9 Author: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Date: Sun Jun 16 15:32:13 2024 +0800 Merge branch 'EvolvingLMMs-Lab:main' into main commit 5bf59ed250da98a408a94e214a73caa400cba842 Author: teowu Date: Sun Jun 16 07:27:28 2024 +0000 fix #117, allow auto download with tar format videos commit 98b3955cb808e36303c030aea78eb037d1ec59ce Merge: a056f118 be9dada8 Author: teowu Date: Sun Jun 16 07:25:07 2024 +0000 Merge branch 'main' of https://github.com/teowu/lmms-eval into main commit a056f118704eccec86ce32ab86981ce4bc1e1deb Author: teowu Date: Sun Jun 16 07:23:54 2024 +0000 fix #117, allow auto download with tar format videos commit 8ba192f94edf5d99598983445d5faa4f8807c49f Merge: 7cc28907 be9dada8 Author: Li Bo Date: Sat Jun 15 17:30:59 2024 +0800 Merge pull request #117 from teowu/main LongVideoBench for LMMs-Eval commit be9dada8b4189c53c08e1674ab273242cf2f80a0 Merge: 62ea8ceb 7cc28907 Author: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Date: Sat Jun 15 16:39:20 2024 +0800 Merge pull request #1 from EvolvingLMMs-Lab/main Merge pull request #113 from teowu/main commit 62ea8ceb223ef2b51ebab2bcd50d5cf339c35cfe Author: teowu Date: Sat Jun 15 08:30:11 2024 +0000 LongVideoBench support: image LMMs (idefics2, phi3) and video LMMs (LLaVA-Next-Video-34B) commit 7cc28907edbb4eb58ee1398772a48110ea35dd96 Merge: 4bc7224d ea14cd4b Author: Li Bo Date: Sat Jun 15 14:10:22 2024 +0800 Merge pull request #113 from teowu/main Q-Bench, Q-Bench2, A-Bench commit dbfb23873979f789477f4797ee2d6071e0fd921e Author: Jingyang Date: Fri Jun 14 16:20:42 2024 -0400 add tinyllava commit ea14cd4b361f4c95b3665cbdb95bc51754090eb5 Author: teowu Date: Fri Jun 14 15:01:52 2024 +0000 Add qbench, qbench2, abench; fix phi3v as its current implementation does not support multi-image commit 4bc7224dcd27fe8b288bfc3fed4d7a9da9635658 Merge: 2797987f bf14cb85 Author: Li Bo Date: Fri Jun 14 02:14:43 2024 +0800 Merge pull request #111 from XinrunDu/main add II-Bench commit bf14cb8527b2b7ac438a36567a875168bc02d294 Author: XinrunDu Date: Thu Jun 13 09:37:02 2024 +0000 fix dataset_path commit 6248113f4e11a0ac396d31fa1b032a142fea8cb4 Author: XinrunDu Date: Thu Jun 13 09:32:06 2024 +0000 add II-Bench commit 2797987f5b88b87bd172714b678a75a1d8051826 Merge: 63d82f1f 66d4bb2d Author: Li Bo Date: Thu Jun 13 11:14:47 2024 +0800 Merge pull request #109 from EvolvingLMMs-Lab/pufanyi/update_version [Small Update] Update the version of LMMs-Eval commit 66d4bb2d9c9afbbdea40196d4ad80e214d0b14b6 Author: Fanyi Pu Date: Thu Jun 13 11:13:00 2024 +0800 update version commit 63d82f1ff11eb430d91a15d6788a1f0b4d596850 Author: Li Bo Date: Thu Jun 13 11:04:32 2024 +0800 Update README.md commit 44a33799671cb668f55366d5e5a4ddb051a3a1b4 Merge: 5ed00356 0ce46d08 Author: Li Bo Date: Thu Jun 13 04:00:12 2024 +0800 Merge pull request #105 from tianyu-z/main Include VCR commit 0ce46d088e473d12d63de44f17c67dceab25658c Author: Suyuchen Date: Wed Jun 12 15:56:34 2024 -0400 update README.md commit 46a88d8b0199ed44d2ff459fb372f2e006960cea Merge: 47b13b9b 5ed00356 Author: Suyuchen Date: Wed Jun 12 15:50:26 2024 -0400 merged readme.md commit 47b13b9b320d36ac53b3622557e31239f7c22621 Author: Suyuchen Date: Wed Jun 12 15:30:52 2024 -0400 update aggregation function for vcr_wiki commit 5ed00356676cf5d0ff056cf27d1b519b8e303ff7 Author: Li Bo Date: Thu Jun 13 03:21:42 2024 +0800 Update README.md commit ed8806839db5988ced672bd162b7b046edb4863a Author: Li Bo Date: Thu Jun 13 03:13:59 2024 +0800 Update README.md commit fea3806026932a6e2bd6e538bcc413e33abdf245 Merge: d99a24ab 05dc8e85 Author: Li Bo Date: Thu Jun 13 03:11:49 2024 +0800 Merge pull request #108 from EvolvingLMMs-Lab/internal_main_dev [Upgrade to v0.2] Embracing Video Evaluations with LMMs-Eval commit 05dc8e853eab7c6bc782a1e2662d2efe7422f767 Author: Bo Li Date: Wed Jun 12 15:56:04 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit cbeee20bc4ffb510a2b23d96cdaf4077be7c2a9e Author: Bo Li Date: Wed Jun 12 15:50:30 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit f00d5498b69dd4f7e54c907ac906abc7c128f000 Author: Bo Li Date: Wed Jun 12 15:46:33 2024 +0000 Update image alignment in README.md commit 34156335db74cef9e3f0915d7172fd6b22456c15 Author: Bo Li Date: Wed Jun 12 15:43:16 2024 +0000 Update llava conv_template in lmms_eval/models/llava.py commit 50575a950736bc8fc1e191310314cbb5fdff5720 Author: Bo Li Date: Wed Jun 12 15:39:03 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit c9b2252fb8a15dd04252af5e6b4613855afd6ada Author: Bo Li Date: Wed Jun 12 15:33:48 2024 +0000 Bump version to 0.2.0.dev0 commit 465bd4205e8097e9c037b24a3ed08dd6a7694efa Merge: e43bd840 d99a24ab Author: Bo Li Date: Wed Jun 12 15:04:25 2024 +0000 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval into internal_main_dev commit e43bd840b63eb499856e36d9d2ba45c924abcead Author: Bo Li Date: Wed Jun 12 14:54:06 2024 +0000 chore: Remove unnecessary files and code related to live_bench and sft_eval tasks commit d99a24abd06df10d07e5a4d0ad5030613f92f2e7 Merge: 374590be a66003be Author: Li Bo Date: Wed Jun 12 19:45:57 2024 +0800 Merge pull request #107 from AtsuMiyai/new_task/upd_update update gpt-3.5-turbo version commit a66003befe4175824a1be6ed59f5f5b88c15f792 Author: AtsuMiyai Date: Wed Jun 12 17:05:17 2024 +0900 update gpt-3.5-turbo version commit ee91f272985f32eeb9cd6faa41afdd8eb49cac30 Author: AtsuMiyai Date: Wed Jun 12 16:50:53 2024 +0900 update gpt-3.5-turbo version commit 326b9694fc77398592b8caf3ba0bc2e2bb903813 Author: tianyu-z Date: Mon Jun 10 20:07:40 2024 -0400 include std and confidence interval commit cd050d4a721d01a2ace0cd030cf7f8dc67eb8c4d Author: Suyuchen Date: Mon Jun 10 18:49:47 2024 -0400 update vcr_wiki tasks in README.md commit 205721e0aad76dde30255e56149bbed121883356 Author: Suyuchen Date: Mon Jun 10 18:43:15 2024 -0400 update vcr_wiki tasks commit db8e718b502469e8536ee359c5559de87635ffc7 Author: tianyu-z Date: Mon Jun 10 16:13:58 2024 -0400 include the try-except logic for spacy commit 427dabb790118f538b64e4e5bf6a7aab9689b3d9 Author: Suyuchen Date: Mon Jun 10 15:51:05 2024 -0400 add crossed_text to vcr_wiki output commit 043b483eb55f7be4fea75c9bc0b9b03d251b109b Author: tianyu-z Date: Mon Jun 10 15:47:00 2024 -0400 switch logic commit e1f04db8f58dd10591fde335ea13f74cda7c79bd Author: tianyu-z Date: Mon Jun 10 02:38:21 2024 -0400 modify the form of VCR commit 96e8d9867c9549ab7490f4b12cfeb6a06238e0aa Author: tianyu-z Date: Mon Jun 10 00:10:30 2024 -0400 init include vcr commit 374590be62f988a76cf6704cfe394cd8ae7d4cb6 Merge: 504685e2 cb3b9ce7 Author: Kaichen Zhang - NTU Date: Fri Jun 7 20:25:48 2024 +0800 Merge pull request #101 from Gumpest/main Update conbench in README commit 504685e20b17659b913cf46f3012c16bf429e09d Author: Li Bo Date: Thu Jun 6 15:42:15 2024 +0800 Update README.md commit cb3b9ce71411da862ff01342a9122a3c656ffbd1 Merge: c9793b38 67b64ea4 Author: Yuan Zhang <56063339+Gumpest@users.noreply.github.com> Date: Thu Jun 6 11:22:24 2024 +0800 Merge branch 'EvolvingLMMs-Lab:main' into main commit c9793b3883714f254a700230b7bee781d6110e73 Author: Yuan Zhang Date: Thu Jun 6 11:21:05 2024 +0800 update README commit 67b64ea44a5a39d96c7a196a8a8345a7486bd912 Merge: 8ee7848a 5fd68451 Author: Li Bo Date: Wed Jun 5 23:12:58 2024 +0800 Merge pull request #100 from Gumpest/main add Conbench commit 5fd684515c55ef643726c1b6c720c7cbd2183ba1 Author: Yuan Zhang Date: Wed Jun 5 21:52:31 2024 +0800 add conbench commit 8ee7848aaa6383aa1f919c3f21199c81db3fff89 Merge: 747e1978 6fefaf7c Author: Li Bo Date: Tue Jun 4 17:09:33 2024 +0800 Merge pull request #95 from AtsuMiyai/new_task/upd add MM-UPD commit 747e19782996065cdce7157ee8c5e15beb5b6c59 Merge: 4854a34d 05843072 Author: Li Bo Date: Tue Jun 4 17:09:04 2024 +0800 Merge pull request #97 from CaraJ7/update Add MathVerse in README.md commit 6fefaf7cea504e35583ee7217449da290295a7a4 Author: AtsuMiyai Date: Tue Jun 4 17:36:39 2024 +0900 update utils.py for leaderboard submission commit 5f4fe360def1c48ea0cb1da6409d192784882308 Author: AtsuMiyai Date: Sun Jun 2 23:28:27 2024 +0900 slightly change query_prompt for the reproduction commit 05843072d608b970bcada1cd0db65a3c80864060 Author: CaraJ7 <1350074492@qq.com> Date: Sun Jun 2 17:05:28 2024 +0800 Add MathVerse in README.md commit 0581ab3cfb362e2024988b46fbbb00324f1233c9 Author: AtsuMiyai Date: Fri May 31 16:09:45 2024 +0900 merge model_specific_prompt_kwargs and dataset_name into each task yaml commit 4854a34d4d37efb5e201f2691ecdb054590cf20b Author: Pu Fanyi Date: Sat May 4 19:23:39 2024 +0800 Group MMMU images into one image (#83) * update * update font * Add matplotlib.font_manager import in utils.py * Refactor font handling in add_order_label function in utils.py * group mmmu --------- Co-authored-by: Li Bo commit d224794c49520f4d28a31862cf977198cd6cbc5e Author: AtsuMiyai Date: Wed May 29 15:15:59 2024 +0900 add upd commit 453e7936424220f02b99517059ca71babfbe5f5a Author: AtsuMiyai Date: Wed May 29 15:03:30 2024 +0900 add upd commit 909edd6769ddcf8a546be4fdd129416687516878 Author: AtsuMiyai Date: Wed May 29 12:52:21 2024 +0900 add upd commit 7c1ac9706cafc4801fa4da181d2f610b7838c7b8 Author: AtsuMiyai Date: Wed May 29 12:50:32 2024 +0900 add upd commit 811301c5280ddd74986645086f026ab730c8848c Author: AtsuMiyai Date: Wed May 29 12:46:58 2024 +0900 add upd commit 71401bafd1d515f704f86ab4817a758542bc4672 Author: AtsuMiyai Date: Wed May 29 12:41:21 2024 +0900 add upd commit 24dc435908d921e9f1a5706e3141b12e5d838d18 Author: Bo Li Date: Mon May 27 10:17:32 2024 +0000 fix compatibility issue of older version llava commit 616edf43731415b35f0f5e97748ed2e017a2891d Author: Bo Li Date: Mon May 27 09:32:26 2024 +0000 [Fix] import issues of multilingual llava and olympiadbench commit 4c5a99e21a63fb0ee1c7d15546d18066e1d9894b Merge: 45c05b2b b05c3e22 Author: Li Bo Date: Mon May 27 14:19:53 2024 +0800 Merge pull request #87 from vfragoso/vifragos/phi3v Adding microsoft/Phi-3-vision-128k-instruct model. commit b05c3e222fabd308dd7af4e04c1c6a0812962fe6 Author: Victor Fragoso Date: Fri May 24 16:36:37 2024 +0000 Adding documentation of Phi3v class. commit c2008971308ce8168d57c24d00b725832f099244 Author: Victor Fragoso Date: Fri May 24 16:25:02 2024 +0000 Adding prompt arguments for Phi3v on MathVista-TestMini commit 7f9fb6bcc6cd24a7b8011b8753d0ea98cc2451fd Author: Victor Fragoso Date: Fri May 24 13:24:16 2024 +0000 Adding Phi3v model. commit 45c05b2b2bece76e06849a52a0d034f9c0ac2367 Author: kcz358 Date: Thu May 23 03:47:36 2024 +0000 Set printing info for llava_hf to debug level commit 53f013ed8278776551ca992562253387cc9968d2 Author: kcz358 Date: Thu May 23 03:41:39 2024 +0000 Fix pope random name in pope full commit 22520a95f13334b75eee0cf0387151067a6bf516 Author: kcz358 Date: Thu May 23 03:41:14 2024 +0000 Add separated pope tasks by category commit d1eefb1565014b47287ffa6b350229062f8f602f Author: kcz358 Date: Thu May 9 08:36:02 2024 +0000 Update gitignore commit b2b4dbd2dc13432c79208db35abf7f55c97f1790 Author: kcz358 Date: Mon May 20 07:45:11 2024 +0000 Comment out Spice in caption task so that don't need to download stanford nlp model commit 662f05ce4c62a46a83f819d3a5925a9bd20059b5 Author: kcz358 Date: Mon May 20 03:13:13 2024 +0000 Comment out parse result in xcomposer commit 09329322916bfbb604d72ddaf50441a0947f8805 Author: kcz358 Date: Thu May 16 03:55:39 2024 +0000 Fix instructblip qformer size mismatch and multi-images problem commit 557a6a3b15e07e506bc05e2cc76ff6a2f8c93964 Author: kcz358 Date: Thu May 16 03:11:41 2024 +0000 Remove redundant code in fuyu commit 6aeb5504e74ed1980b53700d8e4d4dcf7d1b38fc Author: kcz358 Date: Thu May 16 01:45:24 2024 +0000 Fix idefics2 llava in the wild bugs commit aea80e6a71f716951353e1e5d68380243396b4d6 Author: kcz358 Date: Wed May 15 11:07:35 2024 +0000 Better task list_with_num commit 3c12a080d66b9c38f615b961befca7c30f82fa39 Author: Li Bo Date: Sat May 18 02:35:52 2024 +0800 Update LICENSE commit 82317a635a4978b32e095a06cc295d0ae23661c2 Author: Li Bo Date: Sat May 18 02:29:09 2024 +0800 Update LICENSE commit a8bba1cdb51061a0d27bf9a98cca1505b5c58ea5 Author: Li Bo Date: Sat May 18 02:28:03 2024 +0800 Create LICENSE commit caa5893b5fd2c1d32c72b97f371ccd9a8d9ec3a0 Merge: c0944486 423b0060 Author: Li Bo Date: Mon May 13 11:45:26 2024 +0800 Merge pull request #73 from EvolvingLMMs-Lab/kc/qwen_vl_api [Feat] Add qwen vl api commit c09444860362a136f17641f8b2a1f91c2bbc3715 Author: kcz358 Date: Sat May 11 06:11:19 2024 +0000 Fix llava_hf image tokens number issue commit 64f07e497f53e5bcbe9e8fb5830cc7a1daaf7ff1 Author: kcz358 Date: Thu May 9 02:04:10 2024 +0000 Fix endless warning for llava_hf generation commit 8aaa828108da8514dd9cd23a9d6d83a8b67f2d65 Author: Bo Li Date: Thu May 2 06:13:56 2024 +0000 Add model_name parameter to Llava constructor commit 7847dc4d8efe60605102414bb071b1da9851228e Author: kcz358 Date: Tue May 7 03:15:59 2024 +0000 Parse result for llava_hf 1.6 commit 3e56b4f92db39a2ce92903b0c43a34f1d14d59ec Author: kcz358 Date: Tue May 7 03:09:56 2024 +0000 Fix llava_hf generation for 1.6 commit fa3ff92b07ea5aaa633a2039818c310744f84d07 Author: kcz358 Date: Mon May 6 08:32:57 2024 +0000 Fix llava conv template for llama3 commit 423b00606aa77fd6b324c19e3d480b73ab852db6 Author: kcz358 Date: Sun May 5 07:54:52 2024 +0000 Add qwen vl api commit b7fd7a9f7aa3c0e1e50374047dfffc46a7462b90 Merge: 986139a9 c5a130b6 Author: Li Bo Date: Sun May 5 13:19:48 2024 +0800 Merge pull request #59 from EvolvingLMMs-Lab/add_idefics2 add idefics2 commit 986139a9a31154679bdea029b09639f84712db27 Merge: b46239ca 8d3526c0 Author: Li Bo Date: Fri May 3 01:18:18 2024 +0800 Merge pull request #36 from cocoshe/main [Fix] repr llava doc commit b46239cabab7b545ec99d9eae6c851e531b18374 Merge: bc69a744 373265f2 Author: Li Bo Date: Fri May 3 01:17:34 2024 +0800 Merge pull request #56 from gagan3012/main Multilingual LLava bench commit bc69a744d2cffeb06eba62e843bcc7869e27613a Merge: eef3aeb6 626e8a91 Author: Li Bo Date: Fri May 3 01:12:14 2024 +0800 Merge pull request #70 from hunterheiden/hsh/new_task/WebSRC Bugfix: WebSRC should be token-level F1 NOT character-level commit 626e8a91a4af2dd5dd774fc130cc2f4d74b2bc37 Author: Hunter Heidenreich Date: Thu May 2 09:31:03 2024 -0400 Bugfix: WebSRC should be token-level F1 NOT character-level commit eef3aeb6ab589bb1d5045af5b5c1984a69402d19 Merge: c4e9dd9f 9bca4413 Author: Li Bo Date: Thu May 2 14:38:17 2024 +0800 Merge pull request #69 from hunterheiden/hsh/new_task/WebSRC [New Task] WebSRC (multimodal Q&A on web screenshots) commit 9bca441376325173128e5c50087f068e519c48da Author: Hunter Heidenreich Date: Wed May 1 11:07:29 2024 -0400 Add code to enable compilation of submission for WebSRC test split commit 7687495b1ed552eeba088cb9ad5aaf1170e7fff9 Author: Hunter Heidenreich Date: Wed May 1 10:47:32 2024 -0400 Draft and validate websrc eval on dev split commit 4eebd3e5d7ab3b8c3116eea57318db72d2ce32bb Author: Hunter Heidenreich Date: Wed May 1 10:46:54 2024 -0400 Update main README with new task names commit 35fe80b67656114a8824eb59574089663bdc4c9a Author: Hunter Heidenreich Date: Wed May 1 10:46:20 2024 -0400 Draft README for WebSRC commit 955bd0635cc6c14a96ad869f1002e6dbefdc5071 Author: Hunter Heidenreich Date: Tue Apr 30 10:16:21 2024 -0400 Init webSRC commit c4e9dd9f6e40e8586587c4a75987aa109a37f14b Merge: d8a3a99f 319afccb Author: Li Bo Date: Fri Apr 26 14:37:22 2024 +0800 Merge pull request #63 from hunterheiden/hsh/new_task/screenspot New Task: ScreenSpot - Grounding (REC) and instruction generation (REG) on screens commit 319afccbe713ddf40a8a6fa28501e64c0ad34725 Author: Hunter Heidenreich Date: Thu Apr 25 11:44:34 2024 -0400 slight update commit 2f3811ca1bbad6a441016b05fde09a571900fca8 Author: Hunter Heidenreich Date: Thu Apr 25 11:41:04 2024 -0400 Add README file specific to ScreenSpot commit 28962cbe83631ec5d6481aaea4907a7c96fec848 Author: Hunter Heidenreich Date: Wed Apr 24 11:52:33 2024 -0400 Update README to reflect new tasks commit e457cfb4f2d6869e8367d6d5b03ad25ee4acc363 Author: Hunter Heidenreich Date: Tue Apr 23 18:33:16 2024 -0400 Create ScreenSpot on clean branch commit d8a3a99ff6142fe101fa3c188cc7f29593c44345 Merge: 3dcd0158 ed171293 Author: Li Bo Date: Tue Apr 23 10:34:03 2024 +0800 Merge pull request #61 from tupini07/patch-1 Fix typo in Qwen-VL that was causing "reference before assignment" commit ed171293d1e82075c5c6a847fc91ecbfd45cf89f Author: Andrea Tupini Date: Mon Apr 22 14:56:41 2024 -0600 refactor query construction for clarity commit cd874201c46f32a2903ddffae85f9db73e14adfd Author: Andrea Tupini Date: Mon Apr 22 14:54:29 2024 -0600 convert contexts to list if necessary and remove unnecessary construction of `questions` commit 85573674e90c8d505312ba18c5102e0051255078 Author: Andrea Tupini Date: Mon Apr 22 14:47:33 2024 -0600 Fix typo in qwen_vl that was causing "reference before assignment" commit 3dcd01582b719555bcf8eb25d91cc5e42abd2c5f Merge: 95df9fee 743673a1 Author: Li Bo Date: Sat Apr 20 22:03:16 2024 +0800 Merge pull request #60 from CaraJ7/main Add MathVerse commit 743673a1419b6e729e18c96f148745cc739d4c71 Merge: c1a54721 95df9fee Author: CaraJ7 <1350074492@qq.com> Date: Sat Apr 20 21:49:02 2024 +0800 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval commit c1a5472135c3b84061b64d997ab50dda0412ba4f Author: CaraJ7 <1350074492@qq.com> Date: Sat Apr 20 21:45:34 2024 +0800 Add MathVerse commit 373265f24e7a89cbd49ab724a2e388cc0930be78 Author: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Date: Fri Apr 12 17:21:39 2024 -0700 Add files via upload commit d8530514a5ef9378d2adeaceb228b60ec25a6718 Author: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Date: Fri Apr 12 17:19:49 2024 -0700 Create README.md commit 22a4958e993463edff352ac033014f9a485706cc Author: Bo Li Date: Thu Apr 4 17:12:43 2024 +0000 [WIP] adding mmbench dev evaluation (#75) * WIP * Update GPT evaluation model name and sys prompt * 🛠️ Scale accuracy to percentage The accuracy value is now multiplied by 100 in the aggregation function to represent it as a percentage. Regarding the evaluation process, `math` module importation and refactoring reduce progress log verbosity by logging every 100 evaluations instead of 10. It prevents potential logging overflow. Handling of NaN values is added to ensure 'default_value' is set in case of missing data, avoiding errors in split, category, and l2-category assignments. Finally, reporting of categorical and l2-categorical accuracies is streamlined through a new `calculate_hit_rates` function, improving code readability and maintenance. Issue refs: #1427, #1533 * Update GPT evaluation model name and API configuration * Refactor MMBench_Evaluator class to handle missing columns * Add print statements for detailed results in MMBench-CN(CC), MMBench-CN(Dev), and MMBench-EN(Dev) evaluations * Refactor MMBench-CN and MMBench-EN evaluation functions * 🔄 Refactor result processing and logging logic - Simplified the result processing functions across different utility modules (`cc_utils.py`, `cn_utils.py`, `en_utils.py`) to unify the handling of multiple-choice options. Now, all options ("A" to "E") are dynamically added to the result data, and default to "nan" if not provided in the document. - Removed redundant keys directly from the process results dict creation to avoid clutter and align with the new dynamic addition of options. - In `mmbench_evals.py`, removed the unnecessary check for all splits being 'dev' and streamlined the evaluation loop by eliminating the progress bar (tqdm) for a cleaner log output. - Commented-out code and verbose logging during evaluation, which may have interfered with performance, has been removed for a more efficient and less intrusive logging experience. This cleanup reduces redundancy in the codebase and improves evaluation performance. Refs #2045 --------- Co-authored-by: Bo Li (cherry picked from commit a19278c2ea6ddcbca64d3cc7f4efec7fe5775121) commit 8d3526c0869f0ad7747ff6bb02441140792b461c Author: cocoshe <1228759711@qq.com> Date: Thu Mar 28 13:38:36 2024 +0800 fix doc * feat: Add LlavaOneVision model to available models chore: Update sqlitedict dependency to version 2.1.0 * Revert "Squashed commit of the following:" This reverts commit 11b00999df3c43cb225482e030b791b2d454124c. * Refactor available models in lmms_eval Remove duplicate entries for "llava_hf", "llava_onevision", and "longva" in the AVAILABLE_MODELS dictionary in lmms_eval/models/__init__.py. * fix: Handle import errors in lmms_eval models/__init__.py The code changes in this commit fix the handling of import errors in the lmms_eval/models/__init__.py file. Previously, when an import error occurred, the code simply ignored it. This commit updates the code to log an error message using the logger module when an import error occurs. This commit also removes duplicate entries for "llava_hf", "llava_onevision", and "longva" in the AVAILABLE_MODELS dictionary. Recent user commits: - Refactor available models in lmms_eval - Revert "Squashed commit of the following:" - feat: Add LlavaOneVision model to available models - chore: Update sqlitedict dependency to version 2.1.0 * fix: Handle import errors in lmms_eval models/__init__.py * chore: Remove unused imports in lmms_eval/models/__init__.py and lmms_eval/tasks/vcr_wiki/utils.py * Remove unused imports in lmms_eval/tasks/vcr_wiki/utils.py * chore: Update lmms_eval/tasks/vcr_wiki/utils.py This commit updates the `lmms_eval/tasks/vcr_wiki/utils.py` file. It removes unused imports and fixes the condition for loading Spacy models based on the `load_package` value in the config file. Additionally, it adds a debug log message when the Spacy models are not loaded due to `load_package` being set to False. Remove unused imports in `lmms_eval/tasks/vcr_wiki/utils.py` * feat: Add new subtasks to overall score calculation The code changes in this commit add new subtasks to the overall score calculation in the `overall_score` function. The subtasks "ScanQA", "BLINK", "MathVerse", "SciVerse", and "Mantis" are included in the `categories` dictionary. This ensures that the scores for these subtasks are calculated and included in the evaluation results. Remove unused imports and update subtask categories in `utils.py` * feat: Add new subtasks to overall score calculation * chore: Update lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml Update the image aspect ratio in the default template for the llava_interleave_bench task. Change the value of "image_aspect_ratio" from "original" to "pad". This ensures that the generated images have a padded aspect ratio. * if no response directly return 0 * Squashed commit of the following: commit b2a009b6bbf8353172f5a1dd9c29ea1f67610c02 Author: Pu Fanyi Date: Mon Jul 15 19:12:25 2024 -0700 if no response directly return 0 (#142) commit 5fc5f2f5acf454fc99448b0d62eb52b4bffba0d5 Author: Kaichen Zhang - NTU Date: Tue Jul 16 10:12:11 2024 +0800 Add Muirbench (#143) * handle gen kwargs in internvl2 * Add muirbench * Add files via upload (cherry picked from commit 557083a156c3dd67ac79e22b4202e9b69b6b00f4) * update --------- Co-authored-by: Fanyi Pu Co-authored-by: Yan Shu <570533048@qq.com> --- lmms_eval/models/__init__.py | 20 ++- lmms_eval/models/internvl2.py | 2 +- lmms_eval/models/llava_onevision.py | 9 -- lmms_eval/models/vila.py | 17 +-- .../_default_template_interleave_yaml | 2 +- .../tasks/llava_interleave_bench/utils.py | 24 ++++ lmms_eval/tasks/mathvista/mathvista.yaml | 2 +- lmms_eval/tasks/mathvista/mathvista_test.yaml | 7 +- .../tasks/mathvista/mathvista_testmini.yaml | 43 ++---- .../mathvista/mathvista_testmini_cot.yaml | 29 ++++ .../mathvista/mathvista_testmini_format.yaml | 29 ++++ .../mathvista_testmini_solution.yaml | 29 ++++ lmms_eval/tasks/mlvu/mlvu.yaml | 21 +++ lmms_eval/tasks/mlvu/utils.py | 124 ++++++++++++++++++ lmms_eval/tasks/muirbench/utils.py | 2 +- .../olympiadbench/olympiadbench_evals.py | 3 +- .../tasks/vcr_wiki/_default_template_vcr_yaml | 3 +- lmms_eval/tasks/vcr_wiki/utils.py | 50 ++++--- lmms_eval/tasks/vibe_eval/utils.py | 11 +- lmms_eval/tasks/wild_vision_bench/utils.py | 1 + pyproject.toml | 7 +- 21 files changed, 328 insertions(+), 107 deletions(-) create mode 100755 lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml create mode 100755 lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml create mode 100755 lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml create mode 100644 lmms_eval/tasks/mlvu/mlvu.yaml create mode 100644 lmms_eval/tasks/mlvu/utils.py diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py index b77adb484..b618f8a35 100755 --- a/lmms_eval/models/__init__.py +++ b/lmms_eval/models/__init__.py @@ -3,6 +3,9 @@ import hf_transfer from loguru import logger import sys +import hf_transfer + +os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" logger.remove() logger.add(sys.stdout, level="WARNING") @@ -33,9 +36,7 @@ "phi3v": "Phi3v", "tinyllava": "TinyLlava", "llava_hf": "LlavaHf", - "longva": "LongVA", - "llava_onevision": "Llava_OneVision", - "llava_hf": "LlavaHf", + "llava_onevision": "LlavaOneVision", "longva": "LongVA", "vila": "VILA", } @@ -43,9 +44,8 @@ for model_name, model_class in AVAILABLE_MODELS.items(): try: exec(f"from .{model_name} import {model_class}") - except ImportError as e: - # logger.warning(f"Failed to import {model_class} from {model_name}: {e}") - pass + except Exception as e: + logger.debug(f"Failed to import {model_class} from {model_name}: {e}") if os.environ.get("LMMS_EVAL_PLUGINS", None): # Allow specifying other packages to import models from @@ -54,9 +54,5 @@ for model_name, model_class in getattr(m, "AVAILABLE_MODELS").items(): try: exec(f"from {plugin}.models.{model_name} import {model_class}") - except ImportError: - pass - -import hf_transfer - -os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" + except ImportError as e: + logger.debug(f"Failed to import {model_class} from {model_name}: {e}") diff --git a/lmms_eval/models/internvl2.py b/lmms_eval/models/internvl2.py index 730493d9e..5843bdd10 100644 --- a/lmms_eval/models/internvl2.py +++ b/lmms_eval/models/internvl2.py @@ -213,7 +213,7 @@ def generate_until(self, requests) -> List[str]: for k, v in gen_kwargs.items(): if k not in DEFAULT_GEN_KWARGS: pop_keys.append(k) - + for k in pop_keys: gen_kwargs.pop(k) diff --git a/lmms_eval/models/llava_onevision.py b/lmms_eval/models/llava_onevision.py index 3f27c803a..c5c9b5b53 100644 --- a/lmms_eval/models/llava_onevision.py +++ b/lmms_eval/models/llava_onevision.py @@ -54,15 +54,6 @@ except ImportError as e: eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}") -# Import LLaVA-vid modules -try: - from llavavid.model.language_model.llava_qwen import LlavaQwenConfig - from llavavid.model.language_model.llava_llama import LlavaConfig - - AutoConfig.register("llava_qwen", LlavaQwenConfig) - AutoConfig.register("llava_llama", LlavaConfig) -except ImportError as e: - eval_logger.debug(f"LLaVA-vid is not installed. Error: {e}") # Determine best attention implementation if version.parse(torch.__version__) >= version.parse("2.1.2"): diff --git a/lmms_eval/models/vila.py b/lmms_eval/models/vila.py index 295bf123a..5d48af83f 100755 --- a/lmms_eval/models/vila.py +++ b/lmms_eval/models/vila.py @@ -33,9 +33,7 @@ from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria from llava.mm_utils import process_images except ImportError as e: - print(e) - # import pdb;pdb.set_trace() - eval_logger.debug("VILA is not installed. Please install VILA to use this model.") + eval_logger.debug(f"VILA is not installed. Please install VILA to use this model. Error: {e}") @register_model("vila") @@ -81,7 +79,6 @@ def __init__( self.max_frames_num = max_frames_num # self._config = AutoConfig.from_pretrained(self.pretrained) - # import pdb; pdb.set_trace() self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, self.model_name, device_map=self.device_map, attn_implementation=attn_implementation) self.model.image_processor = self._image_processor @@ -202,7 +199,6 @@ def load_video(self, video_path, max_frames_num): return [Image.fromarray(img) for img in spare_frames] except Exception as e: eval_logger.error(f"Failed to load video {video_path} with error: {e}") - # import pdb;pdb.set_trace() return [Image.new("RGB", (448, 448), (0, 0, 0))] * max_frames_num def tok_decode(self, tokens): @@ -278,31 +274,23 @@ def generate_until(self, requests) -> List[str]: pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: - # if self.task_dict[task][split][doc_id]["duration"] != "short": - # # import pdb;pdb.set_trace() - # res.append("A") - # pbar.update(1) - # continue # encode, pad, and truncate contexts for this batch visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] visuals = self.flatten(visuals) num_video_frames = self.model.config.num_video_frames videos = [] - # import pdb;pdb.set_trace() if self.max_frames_num == 0: images = [Image.new("RGB", (448, 448), (0, 0, 0))] * num_video_frames video = process_images(images, self.model.image_processor, self.model.config).half().cuda() videos.append(video) else: for visual in visuals: - # images, video_loading_succeed = LazySupervisedDataset._load_video(visual, num_video_frames, self.model) - # import pdb;pdb.set_trace() if self.video_decode_backend == "decord": images = self.load_video(visual, num_video_frames) elif self.video_decode_backend == "pyav": images = read_video_pyav(visual, num_frm=num_video_frames) - # import pdb;pdb.set_trace() + video = process_images(images, self.model.image_processor, self.model.config).half().cuda() videos.append(video) @@ -370,7 +358,6 @@ def generate_until(self, requests) -> List[str]: outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() print("Question: ", cur_prompt) print("Answer: ", outputs) - # import pdb;pdb.set_trace() res.append(outputs) pbar.update(1) return res diff --git a/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml b/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml index 25b44461f..8c6bbe643 100644 --- a/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml +++ b/lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml @@ -2,7 +2,7 @@ output_type: generate_until generation_kwargs: until: - "ASSISTANT:" - image_aspect_ratio: original + image_aspect_ratio: pad metadata: version: 0.0 api_type : openai diff --git a/lmms_eval/tasks/llava_interleave_bench/utils.py b/lmms_eval/tasks/llava_interleave_bench/utils.py index 77175600b..3c871d3ce 100644 --- a/lmms_eval/tasks/llava_interleave_bench/utils.py +++ b/lmms_eval/tasks/llava_interleave_bench/utils.py @@ -22,6 +22,16 @@ nlrv2 = ["NLVR2_Mantis"] qbench = ["QBench"] +scan_qa = ["ScanQA"] +alfred = ["ALFRED"] +nuscenes = ["nuscenes"] +scannet_chat = ["ScanNet_chat"] +scannet_task = ["ScanNet_task"] +blink = ["BLINK"] +math_verse = ["MathVerse"] +sci_verse = ["SciVerse"] +mantis = ["Mantis"] + def doc_to_visual(doc): max_visual_count = 16 @@ -184,9 +194,19 @@ def overall_score(results): "Puzzle": puzzle, "NLVR2": nlrv2, "QBench": qbench, + "ScanQA": scan_qa, + "ALFRED": alfred, + "nuscenes": nuscenes, + "ScanNet_chat": scannet_chat, + "ScanNet_task": scannet_task, + "BLINK": blink, + "MathVerse": math_verse, + "SciVerse": sci_verse, + "Mantis": mantis, } category_scores = {} + matched_subtasks = set() eval_logger.info(f"Evaluation Sub-Task Results:") for category, subtasks in categories.items(): @@ -196,11 +216,15 @@ def overall_score(results): if result["sub_task"] in subtasks: count += 1 score += result["score"] + matched_subtasks.add(result["sub_task"]) if count > 0: avg_score = score / count category_scores[category] = avg_score eval_logger.info(f"{category}: {avg_score:.3f}") + if not matched_subtasks: + raise ValueError("No subtasks were matched in the results. Check if the subtask names are correct.") + # Calculate overall score total_score = sum(category_scores.values()) num_categories = len(category_scores) diff --git a/lmms_eval/tasks/mathvista/mathvista.yaml b/lmms_eval/tasks/mathvista/mathvista.yaml index b108ca594..24a1b09eb 100755 --- a/lmms_eval/tasks/mathvista/mathvista.yaml +++ b/lmms_eval/tasks/mathvista/mathvista.yaml @@ -4,5 +4,5 @@ task: - mathvista_test metadata: version: 0.0 - gpt_eval_model_name: "gpt-4-0613" + gpt_eval_model_name: "gpt-3.5-turbo" quick_extract: false \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_test.yaml b/lmms_eval/tasks/mathvista/mathvista_test.yaml index 31fc2a45d..35259bd26 100755 --- a/lmms_eval/tasks/mathvista/mathvista_test.yaml +++ b/lmms_eval/tasks/mathvista/mathvista_test.yaml @@ -8,8 +8,6 @@ doc_to_visual: !function utils.mathvista_doc_to_visual doc_to_text: !function utils.mathvista_doc_to_text doc_to_target: "answer" generation_kwargs: - until: - - "ASSISTANT:" max_new_tokens: 1024 temperature: 0 top_p: 1.0 @@ -23,7 +21,4 @@ metric_list: model_specific_prompt_kwargs: default: - shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file + shot_type: "reason-first" # can be "reason-first", "solution", "step-by-step" \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini.yaml index 3f67431ba..289ece3c4 100755 --- a/lmms_eval/tasks/mathvista/mathvista_testmini.yaml +++ b/lmms_eval/tasks/mathvista/mathvista_testmini.yaml @@ -1,34 +1,9 @@ -dataset_path: AI4Math/MathVista -dataset_kwargs: - token: True -task: "mathvista_testmini" -test_split: testmini -output_type: generate_until -doc_to_visual: !function utils.mathvista_doc_to_visual -doc_to_text: !function utils.mathvista_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 1.0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathvista_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathvista_aggregate_results - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step" - shot: 0 - use_caption: False - use_ocr: False - phi3v: - shot_type: "solution" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file +group: mathvista_testmini +task: + - mathvista_testmini_cot + - mathvista_testmini_solution + - mathvista_testmini_format +metadata: + version: 0.0 + gpt_eval_model_name: "gpt-3.5-turbo" + quick_extract: false \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml new file mode 100755 index 000000000..a801dc6db --- /dev/null +++ b/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml @@ -0,0 +1,29 @@ +dataset_path: AI4Math/MathVista +dataset_kwargs: + token: True +task: "mathvista_testmini_cot" +test_split: testmini +output_type: generate_until +doc_to_visual: !function utils.mathvista_doc_to_visual +doc_to_text: !function utils.mathvista_doc_to_text +doc_to_target: "answer" +generation_kwargs: + max_new_tokens: 1024 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false +process_results: !function utils.mathvista_process_results +metric_list: + - metric: gpt_eval_score + aggregation: !function utils.mathvista_aggregate_results + higher_is_better: true + +model_specific_prompt_kwargs: + default: + shot_type: "step-by-step" # can be "reason-first", "solution", "step-by-step" + shot: 0 + use_caption: False + use_ocr: False + phi3v: + shot_type: "solution" \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml new file mode 100755 index 000000000..8dc819716 --- /dev/null +++ b/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml @@ -0,0 +1,29 @@ +dataset_path: AI4Math/MathVista +dataset_kwargs: + token: True +task: "mathvista_testmini_format" +test_split: testmini +output_type: generate_until +doc_to_visual: !function utils.mathvista_doc_to_visual +doc_to_text: !function utils.mathvista_doc_to_text +doc_to_target: "answer" +generation_kwargs: + max_new_tokens: 1024 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false +process_results: !function utils.mathvista_process_results +metric_list: + - metric: gpt_eval_score + aggregation: !function utils.mathvista_aggregate_results + higher_is_better: true + +model_specific_prompt_kwargs: + default: + shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step" + shot: 0 + use_caption: False + use_ocr: False + phi3v: + shot_type: "solution" \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml new file mode 100755 index 000000000..8eb77b88d --- /dev/null +++ b/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml @@ -0,0 +1,29 @@ +dataset_path: AI4Math/MathVista +dataset_kwargs: + token: True +task: "mathvista_testmini_solution" +test_split: testmini +output_type: generate_until +doc_to_visual: !function utils.mathvista_doc_to_visual +doc_to_text: !function utils.mathvista_doc_to_text +doc_to_target: "answer" +generation_kwargs: + max_new_tokens: 1024 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false +process_results: !function utils.mathvista_process_results +metric_list: + - metric: gpt_eval_score + aggregation: !function utils.mathvista_aggregate_results + higher_is_better: true + +model_specific_prompt_kwargs: + default: + shot_type: "solution" # can be "reason-first", "solution", "step-by-step" + shot: 0 + use_caption: False + use_ocr: False + phi3v: + shot_type: "solution" \ No newline at end of file diff --git a/lmms_eval/tasks/mlvu/mlvu.yaml b/lmms_eval/tasks/mlvu/mlvu.yaml new file mode 100644 index 000000000..375da5958 --- /dev/null +++ b/lmms_eval/tasks/mlvu/mlvu.yaml @@ -0,0 +1,21 @@ +dataset_path: sy1998/temp +dataset_kwargs: + token: True + cache_dir: mlvu + video: True +task: mlvu +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mlvu_doc_to_visual +doc_to_text: !function utils.mlvu_doc_to_text +doc_to_target: "answer" +# The return value of process_results will be used by metrics +process_results: !function utils.mlvu_process_results +# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results +metric_list: + - metric: mlvu_percetion_score + aggregation: !function utils.mlvu_aggregate_results + higher_is_better: true + + + diff --git a/lmms_eval/tasks/mlvu/utils.py b/lmms_eval/tasks/mlvu/utils.py new file mode 100644 index 000000000..476cf9dc4 --- /dev/null +++ b/lmms_eval/tasks/mlvu/utils.py @@ -0,0 +1,124 @@ +from collections import defaultdict +import os +import datetime +import json +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file +from pathlib import Path +import yaml +import sys +from typing import List, Dict, Optional, Union +import re +import cv2 +import numpy as np +from loguru import logger as eval_logger + +TASK_TYPES = [ + "TR", + "AR", + "VS", + "NQA", + "ER", + "PQA", + "SSC", + "AO", + "AC" +] + + + +hf_home = os.getenv("HF_HOME", "./~/.cache/huggingface") +base_cache_dir = os.path.expanduser(hf_home) + +with open(Path(__file__).parent / "mlvu.yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) +cache_name = yaml.safe_load("".join(safe_data))["dataset_kwargs"]["cache_dir"] + + + +def mlvu_doc_to_visual(doc): + + cache_dir = os.path.join(base_cache_dir, cache_name) + video_path = doc["video_name"] + video_path = os.path.join(cache_dir, video_path) + if os.path.exists(video_path): + video_path = video_path + else: + sys.exit(f"video path:{video_path} does not exist, please check") + return [video_path] + + +def mlvu_doc_to_text(doc, model_specific_prompt_kwargs=None): + # option_prompt="Carefully watch this video and pay attention to every detail. Based on your observations, select the best option that accurately addresses the question." + option_prompt="" + question = doc["question"] + "\nOnly give the best option.\n" + full_prompt=option_prompt+"\n"+question+"\n"+"Best option: (" + return full_prompt + + +def extract_characters_regex(s): + s = s.strip() + if ")" in s: + index=s.index(")") + pred=s[index-1:index] + return pred + else: + return s + +def mlvu_process_results(doc, results): + """ + Args: + doc: a instance of the eval dataset + results: [pred] + Returns: + a dictionary with key: metric name (in this case videomme score), value: metric value + """ + pred = results[0] + # print("****************",pred) + pred_ans = extract_characters_regex(pred) + + task_type = doc["task_type"] + data_dict = {"question_id": doc["question"], "task_type": task_type, "pred_answer": pred_ans, "answer": doc["answer"]} + + return {f"mlvu_percetion_score": data_dict} + + +def mlvu_aggregate_results(results): + """ + Args: + results: a list of values returned by process_results + Returns: + A score + """ + category2score = {} + for task_type in TASK_TYPES: + category2score[task_type] = {"correct": 0, "answered": 0} + + + for result in results: + task_type = result["task_type"] + category2score[task_type]["answered"] += 1 + category2score[task_type]["correct"] += result["pred_answer"] == result["answer"] + + + for task_cate in TASK_TYPES: + total_correct = 0 + total_answered = 0 + for k, v in category2score.items(): + if task_cate in k: + total_correct += v["correct"] + total_answered += v["answered"] + eval_logger.info(f"Evaluation on Task Categories: {task_cate}: {100 * total_correct / total_answered if total_answered > 0 else 0 : .1f}%") + + total_correct = 0 + total_answered = 0 + for k, v in category2score.items(): + total_correct += v["correct"] + total_answered += v["answered"] + eval_logger.info(f"Overall Performance: {100 * total_correct / total_answered if total_answered > 0 else 0 : .1f}%") + + return 100 * total_correct / total_answered if total_answered > 0 else 0 diff --git a/lmms_eval/tasks/muirbench/utils.py b/lmms_eval/tasks/muirbench/utils.py index 2924edac4..72712d00a 100644 --- a/lmms_eval/tasks/muirbench/utils.py +++ b/lmms_eval/tasks/muirbench/utils.py @@ -114,4 +114,4 @@ def apply(self, resps, docs): # Assuming we need the first response that matches or the original response filtered_resps.append(filtered[0]) - return filtered_resps + return filtered_resps \ No newline at end of file diff --git a/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py b/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py index 9897447fd..41ae7491b 100644 --- a/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py +++ b/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py @@ -1,6 +1,6 @@ import re import sympy as sp - +import math from loguru import logger as eval_logger @@ -9,7 +9,6 @@ from sympy.parsing.latex import parse_latex except ImportError as e: eval_logger.debug("Please install sympy package by running 'pip install sympy' if you want to use OlympiadBenchEvaluator.") -import math # how to use # scorer = OlympiadBenchEvaluator() diff --git a/lmms_eval/tasks/vcr_wiki/_default_template_vcr_yaml b/lmms_eval/tasks/vcr_wiki/_default_template_vcr_yaml index 37ab5e74f..cbe474b4e 100644 --- a/lmms_eval/tasks/vcr_wiki/_default_template_vcr_yaml +++ b/lmms_eval/tasks/vcr_wiki/_default_template_vcr_yaml @@ -14,4 +14,5 @@ generation_kwargs: # The return value of process_results will be used by metrics # Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results metadata: - - version: 0.0.1 \ No newline at end of file + version: 0.0.1 + load_package: False \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/utils.py b/lmms_eval/tasks/vcr_wiki/utils.py index e16f99b4c..5519c0787 100644 --- a/lmms_eval/tasks/vcr_wiki/utils.py +++ b/lmms_eval/tasks/vcr_wiki/utils.py @@ -1,8 +1,7 @@ import datetime +import yaml import json import os -from difflib import SequenceMatcher as SM -from functools import partial import evaluate import numpy as np @@ -10,27 +9,40 @@ from nltk.util import ngrams from spacy.cli import download -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -# Download the English and Chinese models -try: - nlp_en = spacy.load("en_core_web_sm") -except Exception as e: - download("en_core_web_sm") - nlp_en = spacy.load("en_core_web_sm") +from pathlib import Path +from difflib import SequenceMatcher as SM +from functools import partial -try: - nlp_zh = spacy.load("zh_core_web_sm") -except Exception as e: - download("zh_core_web_sm") - nlp_zh = spacy.load("zh_core_web_sm") +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file +from loguru import logger as eval_logger -nlp = {"en": nlp_en, "zh": nlp_zh} -rouge = evaluate.load("rouge") +with open(Path(__file__).parent / "_default_template_vcr_yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) -from loguru import logger as eval_logger + config = yaml.safe_load("".join(safe_data)) -dir_name = os.path.dirname(os.path.abspath(__file__)) +# Download the English and Chinese models +if config["metadata"]["load_package"]: + try: + nlp_en = spacy.load("en_core_web_sm") + nlp_zh = spacy.load("zh_core_web_sm") + nlp = {"en": nlp_en, "zh": nlp_zh} + rouge = evaluate.load("rouge") + except Exception as e: + eval_logger.debug(f"Failed to load spacy models: {e}") + download("en_core_web_sm") + nlp_en = spacy.load("en_core_web_sm") + download("zh_core_web_sm") + nlp_zh = spacy.load("zh_core_web_sm") + eval_logger.debug("Spacy models not loaded due to load_package is False. Please set load_package to True in the config file to load them.") +else: + nlp = {"en": None, "zh": None} + rouge = None aggregate_results_template = { "max_sim_val": 0, diff --git a/lmms_eval/tasks/vibe_eval/utils.py b/lmms_eval/tasks/vibe_eval/utils.py index 9e3b3c562..1ba1cdb71 100644 --- a/lmms_eval/tasks/vibe_eval/utils.py +++ b/lmms_eval/tasks/vibe_eval/utils.py @@ -2,15 +2,18 @@ from dataclasses import dataclass from typing import Optional, List from pathlib import Path -import yaml - -from reka import ChatMessage -from reka.client import Reka +import yaml import re import os from copy import deepcopy +try: + from reka import ChatMessage + from reka.client import Reka +except ImportError: + eval_logger.warning("Reka is not installed, please install it by `pip install reka-api`") + REKA_API_KEY = os.getenv("REKA_API_KEY", "YOUR_API_KEY") with open(Path(__file__).parent / "vibe_eval.yaml", "r") as f: diff --git a/lmms_eval/tasks/wild_vision_bench/utils.py b/lmms_eval/tasks/wild_vision_bench/utils.py index b7ac6c613..f106d0047 100644 --- a/lmms_eval/tasks/wild_vision_bench/utils.py +++ b/lmms_eval/tasks/wild_vision_bench/utils.py @@ -97,6 +97,7 @@ def get_chat_response(base64_image, prompt, max_retries=5, wait_time=10): response = requests.post(API_URL, headers=headers, json=payload, timeout=60) response.raise_for_status() response_data = response.json() + print(response_data) return response_data["choices"][0]["message"]["content"], GPT_EVAL_MODEL_NAME except requests.exceptions.RequestException as e: print(f"Request failed on attempt {attempt+1}: {e}") diff --git a/pyproject.toml b/pyproject.toml index 726ad1648..e01140ea8 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "pytablewriter", "sacrebleu>=1.5.0", "scikit-learn>=0.24.1", - "sqlitedict", + "sqlitedict==2.1.0", "torch>=2.1.0", # to enable sdpa mode for running 34B model on one 80GB GPU "torchvision>=0.16.0", "timm", @@ -83,9 +83,14 @@ vila = [ gemini = [ "google-generativeai", ] +reka = [ + "httpx==0.23.3", + "reka-api", +] all = [ "vila", "gemini", + "reka", ] [tool.setuptools.packages.find] From f96e3e69fe86dcd9cb33d2bc18cc4ff2003de8be Mon Sep 17 00:00:00 2001 From: kcz358 Date: Sun, 21 Jul 2024 02:07:19 +0000 Subject: [PATCH 06/36] Fix llava onevision loglikelihood video bug --- lmms_eval/models/llava_onevision.py | 108 ++++++++++++---------------- 1 file changed, 47 insertions(+), 61 deletions(-) diff --git a/lmms_eval/models/llava_onevision.py b/lmms_eval/models/llava_onevision.py index c5c9b5b53..856aa36cc 100644 --- a/lmms_eval/models/llava_onevision.py +++ b/lmms_eval/models/llava_onevision.py @@ -269,13 +269,37 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: continuation = doc_to_target(self.task_dict[task][split][doc_id]) visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] visuals = self.flatten(visuals) - image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals] if visuals: - image = process_images(visuals, self._image_processor, self._config) - if type(image) is list: - image = [_image.to(dtype=torch.float16, device=self.device) for _image in image] - else: - image = image.to(dtype=torch.float16, device=self.device) + for visual in visuals: + if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default. + self._config.image_aspect_ratio = "pad" + eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") + # if (len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__) and ("image_aspect_ratio" in gen_kwargs.keys()): + # self._config.image_aspect_ratio = gen_kwargs["image_aspect_ratio"] + # eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") + + if type(visual) == PIL.Image.Image: # For image task + image = process_images([visual], self._image_processor, self._config) + if type(image) is list: + image = [_image.to(dtype=torch.float16, device=self.device) for _image in image] + else: + image = image.to(dtype=torch.float16, device=self.device) + + task_type = "image" + + elif type(visual) == str: # For video task + try: + if self.video_decode_backend == "decord": + frames = self.load_video([visual], self.max_frames_num) + elif self.video_decode_backend == "pyav": + frames = read_video_pyav(visual, num_frm=self.max_frames_num) + frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda() + image = frames + except Exception as e: + eval_logger.error(f"Error {e} in loading video") + image = None + + task_type = "video" else: image = None @@ -288,9 +312,14 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: 2. image token is already specified in the context, so we don't need to add it. 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line. """ - image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals) + if task_type == "image": + image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals) if isinstance(visuals, list) else [DEFAULT_IMAGE_TOKEN] + elif task_type == "video": + image_tokens = [DEFAULT_IMAGE_TOKEN] * len(frames) if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN] image_tokens = " ".join(image_tokens) prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts) + else: + question = (contexts[0] if isinstance(contexts, list) else contexts) # This is much safer for llama3, as we now have some object type in it if "llama_3" in self.conv_template: @@ -311,8 +340,18 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: labels = input_ids.clone() # Context part no need to calculate for loss labels[0, : contxt_id.shape[1]] = -100 + + kwargs = {} + if task_type == "image": + kwargs["image_sizes"] = [[visual.size[0], visual.size[1]] for visual in visuals] + elif task_type == "video": + kwargs["modalities"] = ["video"] + self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride + self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode + + with torch.inference_mode(): - outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes) + outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, **kwargs) loss = outputs["loss"] # loss = torch.exp(loss) logits = outputs["logits"] @@ -344,59 +383,6 @@ def load_video(self, video_path, max_frames_num): spare_frames = vr.get_batch(frame_idx).asnumpy() return spare_frames # (frames, height, width, channels) - def preprocess_qwen(self, sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant."): - roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"} - - im_start, im_end = tokenizer.additional_special_tokens_ids - nl_tokens = tokenizer("\n").input_ids - _system = tokenizer("system").input_ids + nl_tokens - _user = tokenizer("user").input_ids + nl_tokens - _assistant = tokenizer("assistant").input_ids + nl_tokens - - # Apply prompt templates - input_ids, targets = [], [] - - source = sources - if roles[source[0]["from"]] != roles["human"]: - source = source[1:] - - input_id, target = [], [] - system = [im_start] + _system + tokenizer(system_message).input_ids + [im_end] + nl_tokens - input_id += system - target += [im_start] + [IGNORE_INDEX] * (len(system) - 3) + [im_end] + nl_tokens - assert len(input_id) == len(target) - for j, sentence in enumerate(source): - role = roles[sentence["from"]] - if has_image and sentence["value"] is not None and "" in sentence["value"]: - num_image = len(re.findall(DEFAULT_IMAGE_TOKEN, sentence["value"])) - texts = sentence["value"].split("") - _input_id = tokenizer(role).input_ids + nl_tokens - for i, text in enumerate(texts): - _input_id += tokenizer(text).input_ids - if i < len(texts) - 1: - _input_id += [IMAGE_TOKEN_INDEX] + nl_tokens - _input_id += [im_end] + nl_tokens - assert sum([i == IMAGE_TOKEN_INDEX for i in _input_id]) == num_image - else: - if sentence["value"] is None: - _input_id = tokenizer(role).input_ids + nl_tokens - else: - _input_id = tokenizer(role).input_ids + nl_tokens + tokenizer(sentence["value"]).input_ids + [im_end] + nl_tokens - input_id += _input_id - if role == "<|im_start|>user": - _target = [im_start] + [IGNORE_INDEX] * (len(_input_id) - 3) + [im_end] + nl_tokens - elif role == "<|im_start|>assistant": - _target = [im_start] + [IGNORE_INDEX] * len(tokenizer(role).input_ids) + _input_id[len(tokenizer(role).input_ids) + 1 : -2] + [im_end] + nl_tokens - else: - raise NotImplementedError - target += _target - - input_ids.append(input_id) - targets.append(target) - input_ids = torch.tensor(input_ids, dtype=torch.long) - targets = torch.tensor(targets, dtype=torch.long) - return input_ids - def generate_until(self, requests: List[Instance]) -> List[str]: res = [] From 2e7fd3f7a01cfd24afd5a70c2ee21ce196823aec Mon Sep 17 00:00:00 2001 From: Pu Fanyi Date: Fri, 26 Jul 2024 07:19:53 -0700 Subject: [PATCH 07/36] LiveBench July (#146) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task --- lmms_eval/tasks/live_bench/live_bench.yaml | 8 +- lmms_eval/tasks/live_bench/utils.py | 43 +- lmms_eval/tasks/mlvu/utils.py | 27 +- lmms_eval/tasks/muirbench/utils.py | 27 +- tools/live_bench/create_dataset.py | 6 +- tools/live_bench/data_summary.ipynb | 112 ++--- tools/live_bench/example.ipynb | 110 +++++ tools/live_bench/filter.ipynb | 18 +- .../live_bench/data_generator/live_bench.py | 28 +- .../data_generator/live_bench_data.py | 10 + .../live_bench/data_generator/prompt.md | 4 +- .../live_bench/data_generator/qa_generator.py | 153 +++++- .../data_generator/question_finalizer.py | 132 +++++ .../live_bench/data_generator/score_getter.py | 10 +- .../live_bench/data_generator/utils/claude.py | 36 +- .../utils/extract_infomation.py | 44 +- .../live_bench/data_generator/utils/gemini.py | 2 +- .../live_bench/data_generator/utils/gpt4v.py | 5 +- tools/live_bench/live_bench/view.ipynb | 393 +++++++++------ .../live_bench/websites/website_list.yaml | 19 +- tools/live_bench/refine_all_results.py | 36 ++ tools/live_bench/script/modify.ipynb | 461 ++++++++++++++++++ tools/live_bench/script/upload_results.py | 85 ++-- 23 files changed, 1376 insertions(+), 393 deletions(-) create mode 100644 tools/live_bench/live_bench/data_generator/question_finalizer.py create mode 100644 tools/live_bench/refine_all_results.py create mode 100644 tools/live_bench/script/modify.ipynb diff --git a/lmms_eval/tasks/live_bench/live_bench.yaml b/lmms_eval/tasks/live_bench/live_bench.yaml index 1612931b1..b4a09853d 100644 --- a/lmms_eval/tasks/live_bench/live_bench.yaml +++ b/lmms_eval/tasks/live_bench/live_bench.yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True task: "live_bench" test_split: test -dataset_name: 2024-06 +dataset_name: 2024-07 output_type: generate_until doc_to_visual: !function utils.livebench_doc_to_visual doc_to_text: !function utils.livebench_doc_to_text @@ -19,11 +19,13 @@ metric_list: - metric: gpt4_eval_score aggregation: !function utils.livebench_aggregate_results higher_is_better: true + # - metric: gpt4_eval_score_mini + # aggregation: !function utils.livebench_aggregate_results + # higher_is_better: true model_specific_prompt_kwargs: default: pre_prompt: "" post_prompt: "" metadata: - version: "2024-06" api_type : openai - gpt_eval_model_name: "gpt-4o" + eval_with_mini: false diff --git a/lmms_eval/tasks/live_bench/utils.py b/lmms_eval/tasks/live_bench/utils.py index ae0b1ac4c..858700295 100644 --- a/lmms_eval/tasks/live_bench/utils.py +++ b/lmms_eval/tasks/live_bench/utils.py @@ -26,8 +26,8 @@ config = yaml.safe_load("".join(safe_data)) -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] API_TYPE = config["metadata"]["api_type"] +EVAL_WITH_MINI = config["metadata"]["eval_with_mini"] if API_TYPE == "openai": API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") @@ -84,7 +84,7 @@ def format_prompt(question, ground_truth_answer, answer, criteria): return _PROMPT_WITH_IMAGE.format(prompt=question, generation=answer, reference=ground_truth_answer, criteria=criteria) -def get_chat_response(base64_images, question, ground_truth_answer, answer, criteria, max_retries=5, wait_time=10): +def get_chat_response(gpt_model_name, base64_images, question, ground_truth_answer, answer, criteria, max_retries=5, wait_time=10): client = openai.OpenAI(api_key=API_KEY) content = [] @@ -114,22 +114,22 @@ def get_chat_response(base64_images, question, ground_truth_answer, answer, crit for attempt in range(max_retries): try: - response = client.chat.completions.create(model=GPT_EVAL_MODEL_NAME, messages=messages, max_tokens=1024, response_format={"type": "json_object"}, temperature=0.0) + response = client.chat.completions.create(model=gpt_model_name, messages=messages, max_tokens=1024, response_format={"type": "json_object"}, temperature=0.0) response_data = response.choices[0].message.content # print(response_data) response_data = json.loads(response_data) rating = response_data["Rating"] explanation = response_data["Explanation"] - return rating, explanation, GPT_EVAL_MODEL_NAME + return rating, explanation, gpt_model_name except requests.exceptions.RequestException as e: eval_logger.warning(f"Request failed on attempt {attempt + 1}: {e}") time.sleep(wait_time) if attempt == max_retries - 1: eval_logger.error(f"Failed to get response after {max_retries} attempts") - return -1, str(e), GPT_EVAL_MODEL_NAME + return -1, str(e), gpt_model_name except Exception as e: eval_logger.error(f"Error on attempt {attempt + 1}: {e}") - return -1, str(e), GPT_EVAL_MODEL_NAME + return -1, str(e), gpt_model_name def image_to_base64(pil_image): @@ -159,19 +159,34 @@ def livebench_doc_to_text(doc, model_specific_prompt_kwargs=None): SUBTASKS = ("Basic Understanding", "Contextual Analysis", "Deeper Implications", "Broader Implications", "Further Insights") -def livebench_process_results(doc, results): +def livebench_process_results_for_name(doc, results, model, eval_name): base64_images = [image_to_base64(image) for image in livebench_doc_to_visual(doc)] subtask = doc["subtask"] criteria = doc["criteria"] if subtask not in SUBTASKS: subtask = "further insights" if not results or results[0] == "": - return {"gpt4_eval_score": {"rating": 0, "explanation": "No response", "model_name": "N/A", "subtask": subtask}} - rating, explanation, model_name = get_chat_response(base64_images=base64_images, question=doc["question"], ground_truth_answer=doc["answer"], answer=results[0] if results else "", criteria=criteria) + return {eval_name: {"rating": 0, "explanation": "No response", "model_name": "N/A", "subtask": subtask}} + rating, explanation, model_name = get_chat_response(gpt_model_name=model, base64_images=base64_images, question=doc["question"], ground_truth_answer=doc["answer"], answer=results[0] if results else "", criteria=criteria) if rating >= 0: - return {"gpt4_eval_score": {"rating": rating, "explanation": explanation, "model_name": model_name, "subtask": subtask, "id": doc["id"]}} + return {eval_name: {"rating": rating, "explanation": explanation, "model_name": model_name, "subtask": subtask, "id": doc["id"]}} else: - return {"gpt4_eval_score": {"rating": -1, "explanation": explanation, "model_name": "N/A", "subtask": subtask, "id": doc["id"]}} + return {eval_name: {"rating": -1, "explanation": explanation, "model_name": "N/A", "subtask": subtask, "id": doc["id"]}} + + +def livebench_process_results_4o(doc, results): + return livebench_process_results_for_name(doc, results, "gpt-4o", "gpt4_eval_score") + + +def livebench_process_results_4o_mini(doc, results): + return livebench_process_results_for_name(doc, results, "gpt-4o-mini", "gpt4_eval_score_mini") + + +def livebench_process_results(doc, results): + res = livebench_process_results_4o(doc, results) + if EVAL_WITH_MINI: + res.update(livebench_process_results_4o_mini(doc, results)) + return res def livebench_aggregate_results(results): @@ -186,9 +201,11 @@ def livebench_aggregate_results(results): count += 1 subtask = result["subtask"] if subtask not in SUBTASKS: - subtask = "further insights" + subtask = "Further Insights" score[result["subtask"]].append(result["rating"] / 10) - res = pd.DataFrame([(subtask, len(score[subtask]), np.mean(score[subtask]) * 100) for subtask in SUBTASKS], columns=["Subtask", "Count", "Average Score"]) + res = [(subtask, len(score[subtask]), np.mean(score[subtask]) * 100) for subtask in SUBTASKS] + res.append(("Total", count, sum_score / count * 100)) + res = pd.DataFrame(res, columns=["Subtask", "Count", "Score"]) print("=" * 50) print(res) print("=" * 50) diff --git a/lmms_eval/tasks/mlvu/utils.py b/lmms_eval/tasks/mlvu/utils.py index 476cf9dc4..046d5423a 100644 --- a/lmms_eval/tasks/mlvu/utils.py +++ b/lmms_eval/tasks/mlvu/utils.py @@ -12,18 +12,7 @@ import numpy as np from loguru import logger as eval_logger -TASK_TYPES = [ - "TR", - "AR", - "VS", - "NQA", - "ER", - "PQA", - "SSC", - "AO", - "AC" -] - +TASK_TYPES = ["TR", "AR", "VS", "NQA", "ER", "PQA", "SSC", "AO", "AC"] hf_home = os.getenv("HF_HOME", "./~/.cache/huggingface") @@ -39,7 +28,6 @@ cache_name = yaml.safe_load("".join(safe_data))["dataset_kwargs"]["cache_dir"] - def mlvu_doc_to_visual(doc): cache_dir = os.path.join(base_cache_dir, cache_name) @@ -54,21 +42,22 @@ def mlvu_doc_to_visual(doc): def mlvu_doc_to_text(doc, model_specific_prompt_kwargs=None): # option_prompt="Carefully watch this video and pay attention to every detail. Based on your observations, select the best option that accurately addresses the question." - option_prompt="" - question = doc["question"] + "\nOnly give the best option.\n" - full_prompt=option_prompt+"\n"+question+"\n"+"Best option: (" + option_prompt = "" + question = doc["question"] + "\nOnly give the best option.\n" + full_prompt = option_prompt + "\n" + question + "\n" + "Best option: (" return full_prompt def extract_characters_regex(s): s = s.strip() if ")" in s: - index=s.index(")") - pred=s[index-1:index] + index = s.index(")") + pred = s[index - 1 : index] return pred else: return s + def mlvu_process_results(doc, results): """ Args: @@ -98,13 +87,11 @@ def mlvu_aggregate_results(results): for task_type in TASK_TYPES: category2score[task_type] = {"correct": 0, "answered": 0} - for result in results: task_type = result["task_type"] category2score[task_type]["answered"] += 1 category2score[task_type]["correct"] += result["pred_answer"] == result["answer"] - for task_cate in TASK_TYPES: total_correct = 0 total_answered = 0 diff --git a/lmms_eval/tasks/muirbench/utils.py b/lmms_eval/tasks/muirbench/utils.py index 72712d00a..5364c5bac 100644 --- a/lmms_eval/tasks/muirbench/utils.py +++ b/lmms_eval/tasks/muirbench/utils.py @@ -1,4 +1,3 @@ - from lmms_eval.filters.extraction import ExtendedRegexFilter from lmms_eval.filters.transformation import MapFilter import re @@ -17,7 +16,7 @@ def muir_doc_to_text(doc, model_specific_prompt_kwargs=None): def muir_doc_to_visual(doc): image_list = [image.convert("RGB") for image in doc["image_list"]] - return image_list + return image_list def muir_doc_to_target(doc): @@ -33,15 +32,15 @@ def muir_process_results(doc, result): image_type = doc["image_type"] data_dict = { - "pred" : pred, - "task" : task, - "idx" : idx, - "image_relation" : image_relation, - "answer" : answer, - "image_type" : image_type, - } + "pred": pred, + "task": task, + "idx": idx, + "image_relation": image_relation, + "answer": answer, + "image_type": image_type, + } - return {"muirbench_score_overall" : data_dict} + return {"muirbench_score_overall": data_dict} def muir_aggregation(results): @@ -62,16 +61,14 @@ def muir_aggregation(results): score = score / len(results) - task_score = {k : v / task_num[k] for k,v in task_score.items()} + task_score = {k: v / task_num[k] for k, v in task_score.items()} print("=" * 50) for k, v in task_score.items(): print(f"{k} : {v:.2f}") print("=" * 50) - - return score - + return score class MultiChoiceRegexFilter(ExtendedRegexFilter): @@ -114,4 +111,4 @@ def apply(self, resps, docs): # Assuming we need the first response that matches or the original response filtered_resps.append(filtered[0]) - return filtered_resps \ No newline at end of file + return filtered_resps diff --git a/tools/live_bench/create_dataset.py b/tools/live_bench/create_dataset.py index c02d811cd..8400d6c69 100644 --- a/tools/live_bench/create_dataset.py +++ b/tools/live_bench/create_dataset.py @@ -4,9 +4,9 @@ if __name__ == "__main__": website = load_websites() - dataset = LiveBench(force_clear=False, name="2024-06") - dataset.capture(websites=website, driver_kwargs={"headless": True}, screen_shoter="single_screen", shoter_kwargs={"screen_size": (1024, 1024)}, qa_generator="gpt4v", scorer="gpt4v", checker="gemini") + dataset = LiveBench() + dataset.capture(websites=website, driver_kwargs={"headless": True}, screen_shoter="single_screen", shoter_kwargs={"screen_size": (1024, 1024)}, qa_generator="gpt4v", scorer="claude", checker="gemini") website = load_websites_from_file("/data/pufanyi/project/lmms-eval/temp/images") - dataset.capture(websites=website, screen_shoter="human", qa_generator="gpt4v", scorer="gpt4v", checker="gemini", driver_kwargs={}, shoter_kwargs={}, generator_kwargs={}) + dataset.capture(websites=website, screen_shoter="human", qa_generator="gpt4v", scorer="claude", checker="gemini", driver_kwargs={}, shoter_kwargs={}, generator_kwargs={}) dataset.upload() diff --git a/tools/live_bench/data_summary.ipynb b/tools/live_bench/data_summary.ipynb index 430481957..291aeff01 100644 --- a/tools/live_bench/data_summary.ipynb +++ b/tools/live_bench/data_summary.ipynb @@ -2,29 +2,30 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/data/pufanyi/anaconda3/anacondabin/envs/live_bench/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" + "Downloading readme: 100%|██████████| 2.45k/2.45k [00:00<00:00, 5.76MB/s]\n", + "Downloading data: 100%|██████████| 182M/182M [00:12<00:00, 14.3MB/s] \n", + "Generating test split: 100%|██████████| 404/404 [00:01<00:00, 364.97 examples/s]\n" ] } ], "source": [ - "from datasets import load_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "data = load_dataset(\"lmms-lab/LiveBench\", \"2024-06\")" + "data = load_dataset(\"lmms-lab/LiveBench\", \"2024-07\")" ] }, { @@ -36,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -48,70 +49,59 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "False" + "subtask\n", + "Contextual Analysis 103\n", + "Deeper Implications 97\n", + "Broader Implications 79\n", + "Basic Understanding 59\n", + "Further Insights 55\n", + "Name: count, dtype: int64" ] }, - "execution_count": 4, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df.iloc[620][\"checker\"] is None" + "df[\"subtask\"].value_counts()" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 18, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "subtask\n", - "Contextual Analysis 477\n", - "Deeper Implications 132\n", - "Basic Understanding 118\n", - "Further Insights 63\n", - "Broader Implications 52\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "df[\"subtask\"].value_counts()" + "df = df.sample(frac=1).reset_index(drop=True)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ - "df = df.sample(frac=1).reset_index(drop=True)" + "df = df[df[\"id\"] != 35]" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/tmp/ipykernel_303711/5529174.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", - " top_50_per_subtask = df.groupby('subtask').apply(lambda x: x.nlargest(50, 'score'))\n" + "/tmp/ipykernel_425178/992929014.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", + " top_50_per_subtask = df.groupby(\"subtask\").apply(lambda x: x.nlargest(50, \"score\"))\n" ] }, { @@ -120,7 +110,7 @@ "250" ] }, - "execution_count": 7, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -133,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -148,7 +138,7 @@ "Name: count, dtype: int64" ] }, - "execution_count": 8, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -159,16 +149,16 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "9.276" + "9.128" ] }, - "execution_count": 9, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -181,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -220,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -232,7 +222,7 @@ "})" ] }, - "execution_count": 11, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -243,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -266,7 +256,7 @@ " 'scorer_name': Value(dtype='string', id=None)}" ] }, - "execution_count": 12, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -284,31 +274,31 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Map: 100%|██████████| 250/250 [00:00<00:00, 273.33 examples/s]it/s]\n", - "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 4.98ba/s]\n", - "Uploading the dataset shards: 100%|██████████| 1/1 [00:18<00:00, 18.34s/it]\n" + "Map: 100%|██████████| 250/250 [00:00<00:00, 314.90 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 5.14ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:19<00:00, 19.84s/it]\n" ] }, { "data": { "text/plain": [ - "CommitInfo(commit_url='https://huggingface.co/datasets/lmms-lab/LiveBench/commit/89ce4e07cf0988b2305b7227ac2677d481e6112a', commit_message='Upload dataset', commit_description='', oid='89ce4e07cf0988b2305b7227ac2677d481e6112a', pr_url=None, pr_revision=None, pr_num=None)" + "CommitInfo(commit_url='https://huggingface.co/datasets/lmms-lab/LiveBench/commit/19dd279ffe05a6911e11f93a1e9699b6daf7be4e', commit_message='Upload dataset', commit_description='', oid='19dd279ffe05a6911e11f93a1e9699b6daf7be4e', pr_url=None, pr_revision=None, pr_num=None)" ] }, - "execution_count": 13, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "data.push_to_hub(\"lmms-lab/LiveBench\", \"2024-06\", split=\"test\")" + "data.push_to_hub(\"lmms-lab/LiveBench\", \"2024-07\", split=\"test\")" ] }, { diff --git a/tools/live_bench/example.ipynb b/tools/live_bench/example.ipynb index 73332ab36..92b9df73e 100644 --- a/tools/live_bench/example.ipynb +++ b/tools/live_bench/example.ipynb @@ -339,6 +339,116 @@ "data = load_dataset(\"lmms-lab/LiveBench\", \"2024-06\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import datasets\n", + "\n", + "data = datasets.load_dataset(\"lmms-lab/LiveBench\", \"2024-07\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " test: Dataset({\n", + " features: ['id', 'images', 'website', 'question', 'answer', 'criteria', 'subtask', 'data_generator', 'checker', 'date_time', 'screen_shoter', 'screen_size', 'score', 'reason', 'scorer_name'],\n", + " num_rows: 250\n", + " })\n", + " train: Dataset({\n", + " features: ['id', 'images', 'website', 'question', 'answer', 'criteria', 'subtask', 'data_generator', 'checker', 'date_time', 'screen_shoter', 'screen_size', 'score', 'reason', 'scorer_name'],\n", + " num_rows: 250\n", + " })\n", + "})" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "data = datasets.DatasetDict({\"test\": data[\"train\"]})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " test: Dataset({\n", + " features: ['id', 'images', 'website', 'question', 'answer', 'criteria', 'subtask', 'data_generator', 'checker', 'date_time', 'screen_shoter', 'screen_size', 'score', 'reason', 'scorer_name'],\n", + " num_rows: 250\n", + " })\n", + "})" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Map: 100%|██████████| 250/250 [00:00<00:00, 305.90 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 5.01ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:02<00:00, 2.10s/it]\n" + ] + }, + { + "data": { + "text/plain": [ + "CommitInfo(commit_url='https://huggingface.co/datasets/lmms-lab/LiveBench/commit/e7641058fcfc9d4a3039db5b995803e492ee5c4e', commit_message='Upload dataset', commit_description='', oid='e7641058fcfc9d4a3039db5b995803e492ee5c4e', pr_url=None, pr_revision=None, pr_num=None)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.push_to_hub(\"lmms-lab/LiveBench\", \"2024-07\")" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/tools/live_bench/filter.ipynb b/tools/live_bench/filter.ipynb index 1a9b59e0f..bc7661f53 100644 --- a/tools/live_bench/filter.ipynb +++ b/tools/live_bench/filter.ipynb @@ -22,21 +22,9 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Downloading readme: 100%|██████████| 1.60k/1.60k [00:00<00:00, 2.29MB/s]\n", - "Downloading data: 100%|██████████| 48.9M/48.9M [00:04<00:00, 11.3MB/s]\n", - "Downloading data: 100%|██████████| 87.6M/87.6M [00:04<00:00, 20.7MB/s]\n", - "Downloading data: 100%|██████████| 103M/103M [00:05<00:00, 18.8MB/s] \n", - "Generating test split: 100%|██████████| 943/943 [00:03<00:00, 290.89 examples/s]\n" - ] - } - ], + "outputs": [], "source": [ - "data = load_dataset(\"lmms-lab/LiveBench\", \"2024-06\")" + "data = load_dataset(\"lmms-lab/LiveBench\", \"2024-07\")" ] }, { @@ -50,7 +38,7 @@ "DatasetDict({\n", " test: Dataset({\n", " features: ['id', 'images', 'website', 'question', 'answer', 'criteria', 'subtask', 'data_generator', 'checker', 'date_time', 'screen_shoter', 'screen_size', 'score', 'reason', 'scorer_name'],\n", - " num_rows: 943\n", + " num_rows: 320\n", " })\n", "})" ] diff --git a/tools/live_bench/live_bench/data_generator/live_bench.py b/tools/live_bench/live_bench/data_generator/live_bench.py index 84f72ec98..41a60d5f0 100644 --- a/tools/live_bench/live_bench/data_generator/live_bench.py +++ b/tools/live_bench/live_bench/data_generator/live_bench.py @@ -13,6 +13,7 @@ from live_bench.data_generator.score_getter import get_score_getter, get_random_score_getter from live_bench.data_generator.response import Response from live_bench.data_generator.utils.extract_infomation import ImageInfomation, InfomationExtractor +from live_bench.data_generator.question_finalizer import QuestionFinalizer import json from typing import List, Tuple @@ -32,13 +33,14 @@ def get_qa_data(images: ScreenImage, qa_generator: QAGenerator, *, infomation_ge def get_live_bench_data( - driver, website: Website, screen_shoter: ScreenShoter, qa_generator: QAGenerator, checker: QAGenerator, infomation_getter: InfomationExtractor, test=False, scorer=None, score_threshold=5 + driver, website: Website, screen_shoter: ScreenShoter, qa_generator: QAGenerator, checker: QAGenerator, infomation_getter: InfomationExtractor, question_finalizer: QuestionFinalizer, test=False, scorer=None, score_threshold=5 ) -> Tuple[List[LiveBenchData], Response]: images = screen_shoter.capture(driver, website) qa_data, logs = get_qa_data(images, qa_generator, test=test, infomation_getter=infomation_getter) data = [] for qa in qa_data: - item = LiveBenchData(screen=images, question=qa.question, answer=qa.answer, subtask=qa.subtask, criteria=qa.criteria, data_generator=qa_generator.get_name(), checker=checker, scorer=scorer) + # qa_data = question_finalizer.finalize_question(qa, images.images) + item = LiveBenchData(screen=images, question=qa.question, answer=qa.answer, subtask=qa.subtask, criteria=qa.criteria, data_generator=qa_generator.get_name(), checker=checker, scorer=scorer, finalizer=question_finalizer) if score_threshold and (not item.score or item.score < score_threshold): continue data.append(item) @@ -94,7 +96,23 @@ def add(self, data: LiveBenchData, id: int = None): organized_data["id"] = id self.hf_data = self.hf_data.add_item(organized_data) - def capture(self, websites: List[Website] = None, *, screen_shoter="single_screen", qa_generator=None, checker=None, driver=None, scorer=None, test=False, driver_kwargs={}, shoter_kwargs={}, generator_kwargs={}, log_folder="./logs"): + def capture( + self, + websites: List[Website] = None, + *, + screen_shoter="single_screen", + qa_generator=None, + checker=None, + driver=None, + scorer=None, + question_finalizer=None, + test=False, + driver_kwargs={}, + shoter_kwargs={}, + generator_kwargs={}, + question_finalizer_kwargs={}, + log_folder="./logs", + ): can_quit_driver = False if driver is None and screen_shoter != "human": driver = load_driver(**driver_kwargs) @@ -112,11 +130,13 @@ def capture(self, websites: List[Website] = None, *, screen_shoter="single_scree scorer = get_score_getter(scorer) elif scorer is None: scorer = get_random_score_getter() + if question_finalizer is None: + question_finalizer = QuestionFinalizer(**question_finalizer_kwargs) logs = [] infomation_getter = InfomationExtractor() for website in tqdm(websites, desc="Capturing websites"): try: - data, log = get_live_bench_data(driver, website, screen_shoter, qa_generator, checker, test=test, scorer=scorer, infomation_getter=infomation_getter) + data, log = get_live_bench_data(driver, website, screen_shoter, qa_generator, checker, test=test, scorer=scorer, infomation_getter=infomation_getter, question_finalizer=question_finalizer) logs.append(log.to_dict()) for d in data: self.add(d) diff --git a/tools/live_bench/live_bench/data_generator/live_bench_data.py b/tools/live_bench/live_bench/data_generator/live_bench_data.py index db04bddbb..3e53f4ca8 100644 --- a/tools/live_bench/live_bench/data_generator/live_bench_data.py +++ b/tools/live_bench/live_bench/data_generator/live_bench_data.py @@ -1,6 +1,7 @@ from live_bench.screen_shoter.screen import ScreenImage from live_bench.data_generator.utils.extract_infomation import ImageInfomation from live_bench.data_generator.qa_generator import QAGenerator +from live_bench.data_generator.question_finalizer import QuestionFinalizer import datasets @@ -40,6 +41,7 @@ def __init__( score: int = None, reason: str = None, checker: QAGenerator = None, + finalizer: QuestionFinalizer = None, scorer_name=None, scorer=None, ): @@ -64,6 +66,14 @@ def __init__( else: self.subtask = subtask self.checker = checker.get_name() + if finalizer: + try: + qa = finalizer.finalize_question(self.question, self.answer, self.criteria, self.screen.images) + except Exception as e: + raise e + self.question = qa["question"] + self.answer = qa["answer"] + self.criteria = qa["criteria"] if self.subtask: for sub in LiveBenchData.SUBTASKS: if sub.lower() in self.subtask.lower(): diff --git a/tools/live_bench/live_bench/data_generator/prompt.md b/tools/live_bench/live_bench/data_generator/prompt.md index 10835d737..faa2fa88b 100644 --- a/tools/live_bench/live_bench/data_generator/prompt.md +++ b/tools/live_bench/live_bench/data_generator/prompt.md @@ -1,4 +1,4 @@ -I would like you to act as a quizmaster who designs questions based on a provided image that would challenge adults to think critically. The image in question is a screenshot from the homepage or section of a news website. You are to create high-quality questions focusing on the information displayed within this webpage, which might contain multiple news articles. Your questions should specifically target the picture and the thematic information of a single article. Your question should be answerable, and checkable. Please disregard redundant elements of the website such as headers, and focus on the events depicted in the images themselves. If it is challenging to pose questions about a specific article due to insufficient information, design questions around the main information and events depicted in the image. +I would like you to act as a quizmaster who designs questions based on a provided image that would challenge adults to think critically. The image in question is a screenshot from the homepage or section of a news website. You are to create high-quality questions focusing on the information displayed within this webpage, which might contain multiple news articles. Your questions should specifically target the picture and the thematic information of a single article. Your question should be answerable, and checkable. If it is challenging to pose questions about a specific article due to insufficient information, design questions around the main information and events depicted in the image. Within your capabilities, try to make the questions more challenging. A well-crafted question about an event should allow respondents to gain deeper insights by observing and analyzing the event, paying attention to the following aspects: @@ -8,7 +8,7 @@ A well-crafted question about an event should allow respondents to gain deeper i - Broader Implications: Questions that extend the discussion beyond the immediate context of the image to its wider impact on society, other fields, or global issues. - Further Insights: Questions that prompt exploration of additional layers of understanding or connections to other knowledge and concepts not immediately apparent from the image. -Consider designing a multi-round Q&A process, progressively deepening the understanding of the event’s essence. Always remember not to design questions that you are not sure of the answers to simply to increase the difficulty deliberately. +Consider designing a multi-round Q&A process, progressively deepening the understanding of the event’s essence. Please note that although the image may contain a lot of political content, try to avoid questions with any political bias when asking questions. Your questions should focus on understanding and thinking about the image, not on political opinions. diff --git a/tools/live_bench/live_bench/data_generator/qa_generator.py b/tools/live_bench/live_bench/data_generator/qa_generator.py index 2ae893ab3..24989277e 100644 --- a/tools/live_bench/live_bench/data_generator/qa_generator.py +++ b/tools/live_bench/live_bench/data_generator/qa_generator.py @@ -2,12 +2,11 @@ import re import os import openai +import anthropic import base64 import json import random import logging -import pathlib -import textwrap import google.generativeai as genai from time import sleep from PIL import Image @@ -16,6 +15,7 @@ from live_bench.data_generator.response import Response from live_bench.screen_shoter import ScreenImage from live_bench.data_generator.utils.gpt4v import format_gpt4v_images, gpt4v_generate_response +from live_bench.data_generator.utils.claude import format_claude_images, claude_generate_response from live_bench.data_generator.utils.gemini import gemini_generate_response from live_bench.data_generator.utils.extract_infomation import InfomationExtractor, ImageInfomation @@ -142,7 +142,7 @@ class GPT4Generator(QAGenerator): def __init__( self, prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), - model="gpt-4-turbo", + model="gpt-4o", example_path=os.path.join(os.path.dirname(__file__), "example"), check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), ): @@ -202,7 +202,8 @@ def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, in example_output_path = os.path.join(self.example_path, "example_output.json") example_image = Image.open(example_image_path) with open(example_output_path, "r") as f: - example_output = f.read() + example_output = json.load(f) + example_output = json.dumps(example_output, indent=4) messages = self.format_messages(images.images, example_image, example_output, infomation) @@ -320,7 +321,9 @@ def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, in example_output_path = os.path.join(self.example_path, "example_output.json") example_image = Image.open(example_image_path) with open(example_output_path, "r") as f: - example_output = f.read() + # example_output = f.read() + example_output = json.load(f) + example_output = json.dumps(example_output, indent=4) messages = self.format_messages(images.images, example_image, example_output, infomation) @@ -377,3 +380,143 @@ def _format_response(self, response: Response) -> List[QAData]: except Exception as e: logger.error(f"Failed to format response: {e}") return [] + + +@register_generator("claude") +class ClaudeGenerator(QAGenerator): + def __init__( + self, + prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), + model="claude-3-5-sonnet-20240620", + example_path=os.path.join(os.path.dirname(__file__), "example"), + check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), + ): + super().__init__(prompt_file) + API_KEY = os.getenv("ANTHROPIC_API_KEY") + if not API_KEY: + raise ValueError("ANTHROPIC_API_KEY environment variable not set.") + self.api_key = API_KEY + self.client = anthropic.Anthropic(api_key=self.api_key) + self.model = model + if os.path.exists(example_path): + self.example_path = example_path + else: + self.example_path = None + if os.path.exists(check_prompt): + with open(check_prompt, "r") as f: + self.check_prompt = f.read() + else: + self.check_prompt = check_prompt + + def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation): + example = [ + { + "type": "text", + "text": "Here are few examples about the task and the expected output format. You can take these as examples to generate your own questions.", + }, + format_claude_images(example_image), + { + "type": "text", + "text": example_output, + }, + ] + content = example + [format_claude_images(image) for image in images] + if infomation: + content.append({"type": "text", "text": str(infomation)}) + content.append( + { + "type": "text", + "text": "Please generate high-quality questions focusing on the information displayed within this webpage. Ensure your response adheres to the examples provided above and is structured in JSON format, incorporating regular expressions to validate the format.", + }, + ) + messages = [ + { + "role": "user", + "content": content, + }, + ] + return messages + + def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation=None, **kwargs): + if self.example_path: + example_image_path = os.path.join(self.example_path, "example_website.png") + example_output_path = os.path.join(self.example_path, "example_output.json") + example_image = Image.open(example_image_path) + with open(example_output_path, "r") as f: + # example_output = f.read() + example_output = json.load(f) + example_output = json.dumps(example_output, indent=4) + + messages = self.format_messages(images.images, example_image, example_output, infomation) + + return claude_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, system=self.prompt, **kwargs) + + def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): + messages = [ + { + "role": "system", + "content": self.check_prompt, + } + ] + content = [] + for img in images: + content.append(format_claude_images(img)) + content.append( + { + "type": "text", + "text": f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}\nSubtask: {subtask}", + }, + ) + if infomation: + content.append( + { + "type": "text", + "text": str(infomation), + }, + ) + content.append( + { + "type": "text", + "text": "Please rephrase or rewrite the high-quality question focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", + }, + ) + messages.append( + { + "role": "user", + "content": content, + } + ) + return messages + + def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, **kwargs): + messages = self.get_check_prompt(question, answer, criteria, subtask, images.images) + return claude_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) + + def format_checked_response(self, response: Response): + data = json.loads(response.content) + question = data.get("question", None) + answer = data.get("answer", None) + criteria = data.get("criteria", None) + subtask = data.get("subtask", None) + return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) + + def _format_response(self, response: Response) -> List[QAData]: + try: + qa_data = [] + content = json.loads(response.content) + for subtask, messages in content.items(): + subtask = subtask.lower() + for message in messages: + message_lower = {k.lower(): v for k, v in message.items()} + try: + question = message_lower["question"] + answer = message_lower["answer"] + criteria = message_lower["criteria"] + qa_data.append(QAData(question=question, answer=answer, criteria=criteria, subtask=subtask)) + except KeyError as e: + logger.error(f"Failed to parse response: {message}") + logger.error(f"Error: {e}") + return qa_data + except Exception as e: + logger.error(f"Failed to format response: {e}") + return [] diff --git a/tools/live_bench/live_bench/data_generator/question_finalizer.py b/tools/live_bench/live_bench/data_generator/question_finalizer.py new file mode 100644 index 000000000..86a625ef8 --- /dev/null +++ b/tools/live_bench/live_bench/data_generator/question_finalizer.py @@ -0,0 +1,132 @@ +import os +import json +import openai +import anthropic +import google.generativeai as genai +from PIL import Image +from typing import List +from abc import ABC, abstractmethod +from live_bench.data_generator.qa_generator import QAData +from live_bench.data_generator.utils.gpt4v import format_gpt4v_images, gpt4v_generate_response +from live_bench.data_generator.utils.claude import format_claude_images, claude_generate_response +from live_bench.data_generator.utils.gemini import gemini_generate_response + + +REJECT_TO_ANSWER = "Reject to answer" + + +class AnswerGetter(ABC): + @abstractmethod + def get_answer(self, question: str, images: List[Image.Image]): + raise NotImplementedError("get_answer not implemented") + + +class GPT4VAnswerGetter(AnswerGetter): + def __init__(self, model: str = "gpt-4o", api_key=None): + self.model = model + if api_key is None: + self.api_key = os.getenv("OPENAI_API_KEY", None) + else: + self.api_key = api_key + self.client = openai.OpenAI(api_key=self.api_key) + + def get_answer(self, question: str, images: List[Image.Image]): + messages = [{"role": "user", "content": format_gpt4v_images(images) + [{"type": "text", "text": question}]}] + response = gpt4v_generate_response(messages=messages, client=self.client, model=self.model) + if response.success: + return response.content + else: + return REJECT_TO_ANSWER + + +class ClaudeAnswerGetter(AnswerGetter): + def __init__(self, model: str = "claude-3-5-sonnet-20240620", api_key=None): + self.model = model + if api_key is None: + self.api_key = os.getenv("ANTHROPIC_API_KEY", None) + else: + self.api_key = api_key + self.client = anthropic.Anthropic(api_key=self.api_key) + + def get_answer(self, question: str, images: List[Image.Image]): + messages = [{"role": "user", "content": format_claude_images(images) + [{"type": "text", "text": question}]}] + response = claude_generate_response(self.client, self.model, messages) + if response.success: + return response.content + else: + return REJECT_TO_ANSWER + + +class GeminiAnswerGetter(AnswerGetter): + def __init__(self, model: str = "gemini-1.5-pro", api_key=None): + self.model = model + self.client = genai.GenerativeModel(model) + + def get_answer(self, question: str, images: List[Image.Image]): + response = gemini_generate_response(self.client, images + [question], max_tokens=2048) + if response.success: + return response.content + else: + return REJECT_TO_ANSWER + + +QUESTION_FINALIZER_PROMPT = """\ +You are a question setter, and your task is to modify the following question, answer, and scoring criteria to ensure: + +1. The question is clear and unambiguous. +2. The answer is correct and reasonable (although the original ground truth answer is mostly correct, it may not be perfect, and sometimes the answer maybe incorrect). +3. The scoring criteria are rational and facilitate the accurate assessment of responses. +4. The full score for the scoring criteria must be 10 points, and it must directly relate to the specific answer. +5. Except for some multiple-choice questions or other questions with only one possible answer, the scoring criteria should not be an all-or-nothing system. Partially correct answers should receive proportional points. +6. Ensure that the scoring system is flexible enough to accommodate slight variations in correct answers while still maintaining a standard for what is considered an acceptable answer. +7. Clearly define what constitutes a full score, partial score, and zero score. +8. The criteria should be as detailed as possible, even if a LLM without image understanding capabilities could still score the answer based on the criteria and ground truth answer correctly. + +Your task is to finalize these standards, thus ensuring the correctness of the answer and the rationality of the scoring criteria. + +Some tips: + +1. For some extremely hard open-ended questions where answers may vary, hitting all points perfectly may not be realistic. In such cases, you can relax the criteria slightly. For example, if there are five possible points in an answer, but answering three adequately could merit full points. An other option is to change the question to a multiple-choice / multi-select question. But remember, it only applies to extremely hard open-ended questions which are impossible to answer perfectly. +2. For some questions, changing the format might be beneficial. You can consider transforming them into different types of questions such as essay, fill-in-the-blank, multiple-choice (or multiple-select), true/false, ranking (e.g., based on time, importance, etc.), or matching questions to enhance the difficulty and rationality of the scoring criteria. +""" + +FINALIZER_OUTPUT_FORMAT_PROMPT = """\ +Please provide the final question, answer, and scoring criteria in the following json format: +{ + "question": "The final question", + "answer": "The final answer", + "criteria": "The final scoring criteria" +} + +One thing as a reminder is that if you want to add a new line in the json string, you should use the escape character "\\n" instead to represent the new line. +""" + + +class QuestionFinalizer(object): + def __init__(self, gpt4v_model: str = "gpt-4o", claude_model: str = "claude-3-5-sonnet-20240620", gemini_model: str = "gemini-1.5-pro"): + self.models = {"GPT4V": GPT4VAnswerGetter(gpt4v_model), "Claude": ClaudeAnswerGetter(claude_model), "Gemini": GeminiAnswerGetter(gemini_model)} + self.client = openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY", None)) + # self.client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY", None)) + + def finalize_question(self, question, answer, criteria, images: List[Image.Image]): + information = [f"[Original Question]\n{question}", f"[Original Answer]\n{answer}", f"[Original Criteria]\n{criteria}"] + information.append( + "Below are answers from three candidates for reference. These answers may not be correct but are reasonably credible (but mostly correct). If any candidate rejects to answer, consider whether there is an issue with the question (such as containing violent or graphic content, or having a clear political bias). If so, please make necessary modifications. For open-ended questions, also consider the reasonableness of these answers. If they are reasonable, you may need to adjust the scoring criteria or the answer itself." + ) + for model_name, model in self.models.items(): + information.append(f"[{model_name} Answer]\n{model.get_answer(question, images)}") + information.append(FINALIZER_OUTPUT_FORMAT_PROMPT) + prompt = "\n\n".join(information) + messages = [{"role": "user", "content": format_gpt4v_images(images) + [{"type": "text", "text": prompt}]}] + try: + response = gpt4v_generate_response(client=self.client, model="gpt-4o", messages=messages, system=QUESTION_FINALIZER_PROMPT) + if response.success: + data = json.loads(response.content) + return { + "question": data["question"], + "answer": data["answer"], + "criteria": data["criteria"], + } + except Exception as e: + print(f"Failed to generate response: {e}") + return None diff --git a/tools/live_bench/live_bench/data_generator/score_getter.py b/tools/live_bench/live_bench/data_generator/score_getter.py index 89553a08e..412989285 100644 --- a/tools/live_bench/live_bench/data_generator/score_getter.py +++ b/tools/live_bench/live_bench/data_generator/score_getter.py @@ -66,7 +66,7 @@ def get_random_score_getter(*args, **kwargs) -> ScoreGetter: @register_score_getter("gpt4v") class GPT4VScoreGetter(ScoreGetter): - def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="gpt-4-turbo", example_path=os.path.join(os.path.dirname(__file__), "example")): + def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="gpt-4o", example_path=os.path.join(os.path.dirname(__file__), "example")): super().__init__() if os.path.exists(prompt): with open(prompt, "r") as f: @@ -91,7 +91,7 @@ def _format_prompt(self, question: str, answer: str, images: List[Image.Image]): for image in images: messages.append(format_gpt4v_images(image)) messages.append({"type": "text", "text": f"Question: {question}\nQuestioner's Answer: {answer}"}) - messages.append({"type": "text", "text": 'You should format you answer into json format like this: {"score": 10, "reason": "some reason"}'}) + messages.append({"type": "text", "text": 'You should format you answer into json format like this: {"reason": "some reason", "score": 10}'}) prompt.append({"role": "user", "content": messages}) return prompt @@ -112,7 +112,7 @@ def get_score(self, question: str, answer: str, images: ScreenImage, *, max_toke @register_score_getter("claude") class ClaudeScoreGetter(ScoreGetter): - def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="claude-3-opus-20240229", example_path=os.path.join(os.path.dirname(__file__), "example")): + def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="claude-3-5-sonnet-20240620", example_path=os.path.join(os.path.dirname(__file__), "example")): super().__init__() if os.path.exists(prompt): with open(prompt, "r") as f: @@ -138,14 +138,14 @@ def _format_prompt(self, question: str, answer: str, images: List[Image.Image]): for image in images: messages.append(format_claude_images(image)) messages.append({"type": "text", "text": f"Question: {question}\nQuestioner's Answer: {answer}"}) - messages.append({"type": "text", "text": 'You should format you answer into json format like this: {"score": 10, "reason": "some reason"}'}) + messages.append({"type": "text", "text": 'You should format you answer into JSON format like this: { "reason": "some reason", "score": 10 }'}) prompt.append({"role": "user", "content": messages}) return prompt def get_score(self, question: str, answer: str, images: ScreenImage, *, max_tokens=4096, max_try_times=5, **kwargs) -> Score: prompt = self._format_prompt(question, answer, images) try: - response = claude_generate_response(self.client, self.model, prompt, self.prompt, max_tokens, max_try_times, **kwargs) + response = claude_generate_response(client=self.client, model=self.model, messages=prompt, system=self.prompt, max_tokens=max_tokens, max_try_times=max_try_times, **kwargs) if response.success: content = json.loads(response.content) score = content.get("score", None) diff --git a/tools/live_bench/live_bench/data_generator/utils/claude.py b/tools/live_bench/live_bench/data_generator/utils/claude.py index 9678b7272..ccd0b381f 100644 --- a/tools/live_bench/live_bench/data_generator/utils/claude.py +++ b/tools/live_bench/live_bench/data_generator/utils/claude.py @@ -2,13 +2,17 @@ import io import base64 from live_bench.data_generator.response import Response +import anthropic import logging from time import sleep +from typing import Union, List logger = logging.getLogger("lmms-eval") -def format_claude_images(image: Image.Image): +def format_claude_images(image: Union[Image.Image, List[Image.Image]]): + if isinstance(image, list): + return [format_claude_images(img) for img in image] buffered = io.BytesIO() image.save(buffered, format="PNG") img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") @@ -22,16 +26,38 @@ def format_claude_images(image: Image.Image): } -def claude_generate_response(client, model, messages, system, max_tokens: int, max_try_times, **kwargs): - messages.append({"role": "assistant", "content": "{"}) +def claude_generate_response(client: anthropic.Anthropic, model, messages, max_tokens: int = 4096, max_try_times=5, system=None, json_format="auto", test=False, **kwargs): + if json_format == "auto": + json_format = False + for message in messages: + if message.get("role") == "user": + contents = message.get("content", []) + if isinstance(contents, str): + if "json" in contents: + json_format = True + break + else: + for content in contents: + if content.get("type", None) == "text" and "json" in content.get("text", ""): + json_format = True + break + + if json_format: + messages.append({"role": "assistant", "content": "{"}) def _generate(): - return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, system=system, **kwargs) + if system: + return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, system=system, **kwargs) + else: + return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, **kwargs) for times in range(max_try_times): try: response = _generate() - return Response(success=True, content="{" + response.content[0].text, full_log={"input": messages, "output": response}) + response_str = response.content[0].text + if json_format: + response_str = "{" + response_str + return Response(success=True, content=response_str, full_log={"input": messages, "output": response.to_dict()}) except Exception as e: logger.error(f"Failed to generate response: {e}") if times < max_try_times - 1: diff --git a/tools/live_bench/live_bench/data_generator/utils/extract_infomation.py b/tools/live_bench/live_bench/data_generator/utils/extract_infomation.py index 6946c815a..f76a6b29c 100644 --- a/tools/live_bench/live_bench/data_generator/utils/extract_infomation.py +++ b/tools/live_bench/live_bench/data_generator/utils/extract_infomation.py @@ -1,9 +1,11 @@ import os +import anthropic import openai import logging from bs4 import BeautifulSoup import requests from live_bench.data_generator.utils.gpt4v import gpt4v_generate_response, format_gpt4v_images +from live_bench.data_generator.utils.claude import claude_generate_response, format_claude_images from live_bench.screen_shoter import ScreenImage from live_bench.websites import Website from live_bench.data_generator.response import Response @@ -11,17 +13,17 @@ logger = logging.getLogger("live-bench") -GPT4V_EXTRACT_TEXT_PROMPT: str = """\ +EXTRACT_TEXT_PROMPT: str = """\ These are the images of the website that we have captured. Please extract the text from the website. You should extract the text from the website as detailed as possible. Only output the text extracted from the website, do not include any other information. """ -GPT4V_FIND_IMAGES_FEATURES_PROMPT: str = """\ +FIND_IMAGES_FEATURES_PROMPT: str = """\ This is a screenshot from a news website. Your task is to identify the meaningful images in this screenshot and extract relevant information about these images, such as the environment depicted, the actions and expressions of the people, and the connection between these images and the corresponding text. You need to think deeply about these images and provide as much detailed and useful information as possible. """ -GPT4V_THINK_DIFFERENTLY_PROMPT: str = """\ +THINK_DIFFERENTLY_PROMPT: str = """\ What makes this website different from other websites? What is special about its news? Since it is a news website, where is the "new" reflected? Do not give a generalized answer; you need to provide detailed answers based on the specific content of each news article and the accompanying illustrations. """ @@ -54,22 +56,28 @@ def get_info(self): class InfomationExtractor(object): - def __init__(self, model="gpt-4-turbo", openai_api_key=None): + def __init__(self, model="claude-3-5-sonnet-20240620", openai_api_key=None, anthropic_api_key=None): + if not anthropic_api_key: + anthropic_api_key = os.getenv("ANTHROPIC_API_KEY", None) if not openai_api_key: - openai_api_key = os.getenv("OPENAI_API_KEY") - if not openai_api_key: - raise ValueError("OPENAI_API_KEY environment variable not set.") - self.client = openai.OpenAI(api_key=openai_api_key) + openai_api_key = os.getenv("OPENAI_API_KEY", None) + if "gpt" in model: + self.client = openai.OpenAI(api_key=openai_api_key) + self.generate_response = gpt4v_generate_response + self.format_images = format_gpt4v_images + elif "claude" in model: + self.client = anthropic.Anthropic(api_key=anthropic_api_key) + self.generate_response = claude_generate_response + self.format_images = format_claude_images self.model = model def extract_text_from_html(self, url): response = requests.get(url) soup = BeautifulSoup(response.text, "html.parser") - text = "\n".join(soup.stripped_strings) return text - def extract_text_from_html_using_gpt4v(self, screen_image: ScreenImage, **kwargs) -> Response: + def extract_text_from_html_from_gpt(self, screen_image: ScreenImage, **kwargs) -> Response: website: Website = screen_image.website if website.url: url = website.url @@ -77,18 +85,18 @@ def extract_text_from_html_using_gpt4v(self, screen_image: ScreenImage, **kwargs text = f"Below is the text extracted from the website {url} for you to take reference:\n{text}" else: text = "" - text = f"{GPT4V_EXTRACT_TEXT_PROMPT}\n{text}" + text = f"{EXTRACT_TEXT_PROMPT}\n{text}" messages = [ { "role": "user", - "content": [{"type": "text", "text": text}] + format_gpt4v_images(screen_image.images), + "content": [{"type": "text", "text": text}] + self.format_images(screen_image.images), } ] - response = gpt4v_generate_response(messages, model=self.model, client=self.client, json_format=False, **kwargs) + response = self.generate_response(messages=messages, model=self.model, client=self.client, json_format=False, **kwargs) return response def extract_infomation(self, screen_image: ScreenImage, **kwargs) -> ImageInfomation: - ocrs = self.extract_text_from_html_using_gpt4v(screen_image) + ocrs = self.extract_text_from_html_from_gpt(screen_image) infomation = ImageInfomation() if ocrs.success: ocrs = f"Below is the text extracted from the website for you to take reference:\n{ocrs.content}" @@ -98,19 +106,19 @@ def extract_infomation(self, screen_image: ScreenImage, **kwargs) -> ImageInfoma messages = [ { "role": "user", - "content": [{"type": "text", "text": f"{GPT4V_FIND_IMAGES_FEATURES_PROMPT}\n{ocrs}"}] + format_gpt4v_images(screen_image.images), + "content": [{"type": "text", "text": f"{FIND_IMAGES_FEATURES_PROMPT}\n{ocrs}"}] + self.format_images(screen_image.images), } ] - response = gpt4v_generate_response(messages, model=self.model, client=self.client, json_format=False, **kwargs) + response = self.generate_response(messages=messages, model=self.model, client=self.client, json_format=False, **kwargs) if response.success: infomation.image_features = response.content messages = [ { "role": "user", - "content": [{"type": "text", "text": f"{GPT4V_THINK_DIFFERENTLY_PROMPT}\n\n{str(infomation)}"}] + format_gpt4v_images(screen_image.images), + "content": [{"type": "text", "text": f"{THINK_DIFFERENTLY_PROMPT}\n\n{str(infomation)}"}] + self.format_images(screen_image.images), } ] - response = gpt4v_generate_response(messages, model=self.model, client=self.client, json_format=False, **kwargs) + response = self.generate_response(messages=messages, model=self.model, client=self.client, json_format=False, **kwargs) if response.success: infomation.differnt_points = response.content return infomation diff --git a/tools/live_bench/live_bench/data_generator/utils/gemini.py b/tools/live_bench/live_bench/data_generator/utils/gemini.py index 99c1eef6e..57b77536c 100644 --- a/tools/live_bench/live_bench/data_generator/utils/gemini.py +++ b/tools/live_bench/live_bench/data_generator/utils/gemini.py @@ -7,7 +7,7 @@ logger = logging.getLogger("lmms-eval") -def gemini_generate_response(client: genai.GenerativeModel, messages, max_tokens: int, max_try_times, **kwargs): +def gemini_generate_response(client: genai.GenerativeModel, messages, max_tokens: int, max_try_times: int = 5, **kwargs): generation_config = genai.GenerationConfig(max_output_tokens=max_tokens) def _generate(): diff --git a/tools/live_bench/live_bench/data_generator/utils/gpt4v.py b/tools/live_bench/live_bench/data_generator/utils/gpt4v.py index 2ee707949..270a76ea1 100644 --- a/tools/live_bench/live_bench/data_generator/utils/gpt4v.py +++ b/tools/live_bench/live_bench/data_generator/utils/gpt4v.py @@ -34,7 +34,10 @@ def format_printable_messages(messages): return messages -def gpt4v_generate_response(messages, *, client=None, model="gpt-4-turbo", max_tokens: int = 4096, max_try_times: int = 5, json_format="auto", test=False, **kwargs) -> Response: +def gpt4v_generate_response(messages, *, client=None, model="gpt-4o", max_tokens: int = 4096, max_try_times: int = 5, json_format="auto", test=False, system=None, **kwargs) -> Response: + if system: + messages = [{"role": "system", "content": system}] + messages + if json_format == "auto": json_format = False for message in messages: diff --git a/tools/live_bench/live_bench/view.ipynb b/tools/live_bench/live_bench/view.ipynb index 71d99d91a..5a4addacf 100644 --- a/tools/live_bench/live_bench/view.ipynb +++ b/tools/live_bench/live_bench/view.ipynb @@ -2,27 +2,30 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Downloading data: 100%|██████████| 470k/470k [00:00<00:00, 575kB/s]\n", - "Generating test split: 100%|██████████| 9/9 [00:00<00:00, 341.15 examples/s]\n" + "/data/pufanyi/anaconda3/anacondabin/envs/live_bench/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "Downloading readme: 100%|██████████| 2.45k/2.45k [00:00<00:00, 5.08MB/s]\n", + "Downloading data: 100%|██████████| 88.6M/88.6M [00:04<00:00, 19.6MB/s]\n", + "Generating test split: 100%|██████████| 320/320 [00:00<00:00, 384.23 examples/s]\n" ] } ], "source": [ "from datasets import load_dataset\n", "\n", - "dataset = load_dataset(\"lmms-lab/LiveBench\", \"2024-06\")" + "dataset = load_dataset(\"lmms-lab/LiveBench\", \"2024-07\")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -30,13 +33,13 @@ "text/plain": [ "DatasetDict({\n", " test: Dataset({\n", - " features: ['id', 'images', 'website', 'question', 'answer', 'subtask', 'data_generator', 'checker', 'date_time', 'screen_shoter', 'screen_size', 'score', 'reason', 'scorer_name'],\n", - " num_rows: 9\n", + " features: ['id', 'images', 'website', 'question', 'answer', 'criteria', 'subtask', 'data_generator', 'checker', 'date_time', 'screen_shoter', 'screen_size', 'score', 'reason', 'scorer_name'],\n", + " num_rows: 320\n", " })\n", "})" ] }, - "execution_count": 5, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -47,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -76,6 +79,7 @@ " website\n", " question\n", " answer\n", + " criteria\n", " subtask\n", " data_generator\n", " checker\n", @@ -92,228 +96,301 @@ " 0\n", " 0\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Look at the image provided. Two of the article...\n", - " The shared theme is the legal situation of Hun...\n", - " Contextual Analysis\n", - " gpt4v\n", + " {'url': 'https://www.bbc.com/news'}\n", + " Examine the top menu bar of the BBC website di...\n", + " C) Weather\n", + " Award 10 points for selecting option C) Weathe...\n", + " Basic Understanding\n", + " claude\n", " gemini\n", - " 2024-06-12 01:14:15\n", + " 2024-07-20 14:02:22\n", " single_screen\n", " (1024, 1024)\n", " 10\n", - " The answer accurately identifies the article d...\n", - " gpt4v\n", + " The answer is correct and can be directly veri...\n", + " claude\n", " \n", " \n", " 1\n", " 1\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Based on the provided image, what is the main ...\n", - " Ukraine has launched missiles at sites within ...\n", - " Basic Understanding\n", - " gpt4v\n", + " {'url': 'https://www.bbc.com/news'}\n", + " Based on the image accompanying the article ab...\n", + " The image depicts a crowded airport setting, l...\n", + " {'10 points': 'The answer correctly identifies...\n", + " Deeper Implications\n", + " claude\n", " gemini\n", - " 2024-06-12 01:14:15\n", + " 2024-07-20 14:02:22\n", " single_screen\n", " (1024, 1024)\n", " 10\n", - " The information provided in the answer directl...\n", - " gpt4v\n", + " The image clearly shows that the news article ...\n", + " claude\n", " \n", " \n", " 2\n", " 2\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Based on the juxtaposition of the articles on ...\n", - " The two narratives are the personal and legal ...\n", + " {'url': 'https://www.bbc.com/news'}\n", + " Based on the photograph accompanying the headl...\n", + " The photograph shows a crowded scene inside an...\n", + " {'2 points': 'Interprets the overall scene as ...\n", " Contextual Analysis\n", - " gpt4v\n", + " claude\n", " gemini\n", - " 2024-06-12 01:14:15\n", + " 2024-07-20 14:02:22\n", " single_screen\n", " (1024, 1024)\n", - " 8\n", - " The answer is mostly authentic as the convicti...\n", - " gpt4v\n", + " 10\n", + " The answer accurately describes the image and ...\n", + " claude\n", " \n", " \n", " 3\n", " 3\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Examining the headline and image, what visual ...\n", - " The image showcasing a relatively unscathed st...\n", - " Contextual Analysis\n", - " gpt4v\n", - " gemini\n", - " 2024-06-12 01:14:15\n", + " {'url': 'https://www.bbc.com/news'}\n", + " Considering the headline 'Global services slow...\n", + " This incident underscores several critical iss...\n", + " Award up to 10 marks: 1.5 points for discussin...\n", + " Deeper Implications\n", + " claude\n", + " None\n", + " 2024-07-20 14:02:22\n", " single_screen\n", " (1024, 1024)\n", - " 8\n", - " The answer is relevant and provides a plausibl...\n", - " gpt4v\n", + " 10\n", + " The answer provides an insightful analysis of ...\n", + " claude\n", " \n", " \n", " 4\n", " 4\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Based on the juxtaposition of the articles on ...\n", - " The audience might infer a potential media bia...\n", - " Deeper Implications\n", - " gpt4v\n", + " {'url': 'https://www.bbc.com/news'}\n", + " Analyze the layout and image selection of the ...\n", + " The layout of the BBC News homepage utilizes v...\n", + " {'10 points': 'The answer provides a detailed ...\n", + " Contextual Analysis\n", + " claude\n", " gemini\n", - " 2024-06-12 01:14:15\n", + " 2024-07-20 14:02:22\n", " single_screen\n", " (1024, 1024)\n", " 8\n", - " The response provides a reasoned inference bas...\n", - " gpt4v\n", + " The ranking provided is generally accurate and...\n", + " claude\n", " \n", " \n", - " 5\n", - " 5\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " \n", + " \n", + " 315\n", + " 315\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Examining the image and the headline \"What son...\n", - " Potential narratives could include: the impact...\n", - " Broader Implications\n", + " {'name': 'bloomberg.comeconomics.png'}\n", + " Analyze the image accompanying the article 'Xi...\n", + " The image accompanying the article suggests a ...\n", + " Scoring Criteria (Total: 10 points)\\n\\n- Ident...\n", + " Contextual Analysis\n", " gpt4v\n", " gemini\n", - " 2024-06-12 01:14:15\n", - " single_screen\n", - " (1024, 1024)\n", - " 7\n", - " Authenticity (4/5): The information about it b...\n", - " gpt4v\n", + " 2024-07-21 20:23:39\n", + " human\n", + " None\n", + " 6\n", + " While the question is thoughtful and relevant ...\n", + " claude\n", " \n", " \n", - " 6\n", - " 6\n", + " 316\n", + " 316\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Based on the visual framing and placement of t...\n", - " The BBC seemingly prioritizes Hunter Biden's t...\n", + " {'name': 'wsj.comworldafrica.png'}\n", + " Analyze the image accompanying the article 'Qu...\n", + " The image contains several key elements that s...\n", + " Scoring Criteria (Total: 10 points):\\n\\n1. Ide...\n", " Contextual Analysis\n", " gpt4v\n", " gemini\n", - " 2024-06-12 01:14:15\n", - " single_screen\n", - " (1024, 1024)\n", - " 8\n", - " The answer effectively compares domestic polit...\n", - " gpt4v\n", + " 2024-07-21 20:27:57\n", + " human\n", + " None\n", + " 6\n", + " While the question is based on information pro...\n", + " claude\n", " \n", " \n", - " 7\n", - " 7\n", + " 317\n", + " 317\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Considering the use of the word \"LIVE\" in the ...\n", - " The use of \"LIVE\" could lead the public to per...\n", - " Deeper Implications\n", + " {'name': 'wsj.comworldafrica.png'}\n", + " Based on the articles shown on the Wall Street...\n", + " The two most closely related news stories are:...\n", + " Scoring Criteria (Total 10 points):\\n1. Correc...\n", + " Contextual Analysis\n", " gpt4v\n", " gemini\n", - " 2024-06-12 01:14:15\n", - " single_screen\n", - " (1024, 1024)\n", - " 10\n", - " The answer provided is directly related to the...\n", - " gpt4v\n", + " 2024-07-21 20:27:57\n", + " human\n", + " None\n", + " 7\n", + " This question requires analysis and interpreta...\n", + " claude\n", " \n", " \n", - " 8\n", - " 8\n", + " 318\n", + " 318\n", " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", - " {'url': 'https://www.bbc.com/'}\n", - " Based on the image alone, what elements sugges...\n", - " The juxtaposition of the image depicting Presi...\n", - " Deeper Implications\n", + " {'name': 'wsj.comworldafrica.png'}\n", + " Analyze the image and headlines related to the...\n", + " The image and headlines present a complex narr...\n", + " Scoring Criteria (Total: 10 points)\\n\\n1. Anal...\n", + " Contextual Analysis\n", " gpt4v\n", " gemini\n", - " 2024-06-12 01:14:15\n", - " single_screen\n", - " (1024, 1024)\n", - " 8\n", - " The answer effectively discusses the potential...\n", + " 2024-07-21 20:27:57\n", + " human\n", + " None\n", + " 6\n", + " This question and answer are not directly supp...\n", + " claude\n", + " \n", + " \n", + " 319\n", + " 319\n", + " [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH...\n", + " {'name': 'wsj.comworldafrica.png'}\n", + " Analyze the image accompanying the article tit...\n", + " The image contains several key visual elements...\n", + " Scoring Criteria (Total 10 points):\\n\\n- Ident...\n", + " Basic Understanding\n", " gpt4v\n", + " gemini\n", + " 2024-07-21 20:27:57\n", + " human\n", + " None\n", + " 5\n", + " This answer is not directly supported by the i...\n", + " claude\n", " \n", " \n", "\n", + "

320 rows × 15 columns

\n", "" ], "text/plain": [ - " id images \\\n", - "0 0 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "1 1 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "2 2 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "3 3 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "4 4 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "5 5 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "6 6 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "7 7 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", - "8 8 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + " id images \\\n", + "0 0 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "1 1 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "2 2 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "3 3 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "4 4 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + ".. ... ... \n", + "315 315 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "316 316 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "317 317 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "318 318 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "319 319 [{'bytes': b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIH... \n", + "\n", + " website \\\n", + "0 {'url': 'https://www.bbc.com/news'} \n", + "1 {'url': 'https://www.bbc.com/news'} \n", + "2 {'url': 'https://www.bbc.com/news'} \n", + "3 {'url': 'https://www.bbc.com/news'} \n", + "4 {'url': 'https://www.bbc.com/news'} \n", + ".. ... \n", + "315 {'name': 'bloomberg.comeconomics.png'} \n", + "316 {'name': 'wsj.comworldafrica.png'} \n", + "317 {'name': 'wsj.comworldafrica.png'} \n", + "318 {'name': 'wsj.comworldafrica.png'} \n", + "319 {'name': 'wsj.comworldafrica.png'} \n", + "\n", + " question \\\n", + "0 Examine the top menu bar of the BBC website di... \n", + "1 Based on the image accompanying the article ab... \n", + "2 Based on the photograph accompanying the headl... \n", + "3 Considering the headline 'Global services slow... \n", + "4 Analyze the layout and image selection of the ... \n", + ".. ... \n", + "315 Analyze the image accompanying the article 'Xi... \n", + "316 Analyze the image accompanying the article 'Qu... \n", + "317 Based on the articles shown on the Wall Street... \n", + "318 Analyze the image and headlines related to the... \n", + "319 Analyze the image accompanying the article tit... \n", "\n", - " website \\\n", - "0 {'url': 'https://www.bbc.com/'} \n", - "1 {'url': 'https://www.bbc.com/'} \n", - "2 {'url': 'https://www.bbc.com/'} \n", - "3 {'url': 'https://www.bbc.com/'} \n", - "4 {'url': 'https://www.bbc.com/'} \n", - "5 {'url': 'https://www.bbc.com/'} \n", - "6 {'url': 'https://www.bbc.com/'} \n", - "7 {'url': 'https://www.bbc.com/'} \n", - "8 {'url': 'https://www.bbc.com/'} \n", + " answer \\\n", + "0 C) Weather \n", + "1 The image depicts a crowded airport setting, l... \n", + "2 The photograph shows a crowded scene inside an... \n", + "3 This incident underscores several critical iss... \n", + "4 The layout of the BBC News homepage utilizes v... \n", + ".. ... \n", + "315 The image accompanying the article suggests a ... \n", + "316 The image contains several key elements that s... \n", + "317 The two most closely related news stories are:... \n", + "318 The image and headlines present a complex narr... \n", + "319 The image contains several key visual elements... \n", "\n", - " question \\\n", - "0 Look at the image provided. Two of the article... \n", - "1 Based on the provided image, what is the main ... \n", - "2 Based on the juxtaposition of the articles on ... \n", - "3 Examining the headline and image, what visual ... \n", - "4 Based on the juxtaposition of the articles on ... \n", - "5 Examining the image and the headline \"What son... \n", - "6 Based on the visual framing and placement of t... \n", - "7 Considering the use of the word \"LIVE\" in the ... \n", - "8 Based on the image alone, what elements sugges... \n", + " criteria subtask \\\n", + "0 Award 10 points for selecting option C) Weathe... Basic Understanding \n", + "1 {'10 points': 'The answer correctly identifies... Deeper Implications \n", + "2 {'2 points': 'Interprets the overall scene as ... Contextual Analysis \n", + "3 Award up to 10 marks: 1.5 points for discussin... Deeper Implications \n", + "4 {'10 points': 'The answer provides a detailed ... Contextual Analysis \n", + ".. ... ... \n", + "315 Scoring Criteria (Total: 10 points)\\n\\n- Ident... Contextual Analysis \n", + "316 Scoring Criteria (Total: 10 points):\\n\\n1. Ide... Contextual Analysis \n", + "317 Scoring Criteria (Total 10 points):\\n1. Correc... Contextual Analysis \n", + "318 Scoring Criteria (Total: 10 points)\\n\\n1. Anal... Contextual Analysis \n", + "319 Scoring Criteria (Total 10 points):\\n\\n- Ident... Basic Understanding \n", "\n", - " answer subtask \\\n", - "0 The shared theme is the legal situation of Hun... Contextual Analysis \n", - "1 Ukraine has launched missiles at sites within ... Basic Understanding \n", - "2 The two narratives are the personal and legal ... Contextual Analysis \n", - "3 The image showcasing a relatively unscathed st... Contextual Analysis \n", - "4 The audience might infer a potential media bia... Deeper Implications \n", - "5 Potential narratives could include: the impact... Broader Implications \n", - "6 The BBC seemingly prioritizes Hunter Biden's t... Contextual Analysis \n", - "7 The use of \"LIVE\" could lead the public to per... Deeper Implications \n", - "8 The juxtaposition of the image depicting Presi... Deeper Implications \n", + " data_generator checker date_time screen_shoter screen_size \\\n", + "0 claude gemini 2024-07-20 14:02:22 single_screen (1024, 1024) \n", + "1 claude gemini 2024-07-20 14:02:22 single_screen (1024, 1024) \n", + "2 claude gemini 2024-07-20 14:02:22 single_screen (1024, 1024) \n", + "3 claude None 2024-07-20 14:02:22 single_screen (1024, 1024) \n", + "4 claude gemini 2024-07-20 14:02:22 single_screen (1024, 1024) \n", + ".. ... ... ... ... ... \n", + "315 gpt4v gemini 2024-07-21 20:23:39 human None \n", + "316 gpt4v gemini 2024-07-21 20:27:57 human None \n", + "317 gpt4v gemini 2024-07-21 20:27:57 human None \n", + "318 gpt4v gemini 2024-07-21 20:27:57 human None \n", + "319 gpt4v gemini 2024-07-21 20:27:57 human None \n", "\n", - " data_generator checker date_time screen_shoter screen_size \\\n", - "0 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "1 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "2 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "3 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "4 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "5 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "6 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "7 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", - "8 gpt4v gemini 2024-06-12 01:14:15 single_screen (1024, 1024) \n", + " score reason scorer_name \n", + "0 10 The answer is correct and can be directly veri... claude \n", + "1 10 The image clearly shows that the news article ... claude \n", + "2 10 The answer accurately describes the image and ... claude \n", + "3 10 The answer provides an insightful analysis of ... claude \n", + "4 8 The ranking provided is generally accurate and... claude \n", + ".. ... ... ... \n", + "315 6 While the question is thoughtful and relevant ... claude \n", + "316 6 While the question is based on information pro... claude \n", + "317 7 This question requires analysis and interpreta... claude \n", + "318 6 This question and answer are not directly supp... claude \n", + "319 5 This answer is not directly supported by the i... claude \n", "\n", - " score reason scorer_name \n", - "0 10 The answer accurately identifies the article d... gpt4v \n", - "1 10 The information provided in the answer directl... gpt4v \n", - "2 8 The answer is mostly authentic as the convicti... gpt4v \n", - "3 8 The answer is relevant and provides a plausibl... gpt4v \n", - "4 8 The response provides a reasoned inference bas... gpt4v \n", - "5 7 Authenticity (4/5): The information about it b... gpt4v \n", - "6 8 The answer effectively compares domestic polit... gpt4v \n", - "7 10 The answer provided is directly related to the... gpt4v \n", - "8 8 The answer effectively discusses the potential... gpt4v " + "[320 rows x 15 columns]" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } diff --git a/tools/live_bench/live_bench/websites/website_list.yaml b/tools/live_bench/live_bench/websites/website_list.yaml index 66f7cabdc..c85605d7d 100644 --- a/tools/live_bench/live_bench/websites/website_list.yaml +++ b/tools/live_bench/live_bench/websites/website_list.yaml @@ -6,13 +6,13 @@ websites: # args: # arg1: value1 # arg2: value2 -- url: https://www.bbc.com/news -- url: https://www.bbc.com/sport -- url: https://www.bbc.com/business -- url: https://www.bbc.com/innovation -- url: https://www.bbc.com/culture -- url: https://www.bbc.com/travel -- url: https://www.bbc.com/future-planet +# - url: https://www.bbc.com/news +# - url: https://www.bbc.com/sport +# - url: https://www.bbc.com/business +# - url: https://www.bbc.com/innovation +# - url: https://www.bbc.com/culture +# - url: https://www.bbc.com/travel +# - url: https://www.bbc.com/future-planet # - url: https://edition.cnn.com/ # - url: https://edition.cnn.com/politics # - url: https://edition.cnn.com/entertainment @@ -74,6 +74,5 @@ websites: # - url: https://news.ycombinator.com/ # - url: https://www.reddit.com/?rdt=48006 # - url: https://news.crunchbase.com/ -- url: https://www.cctv.com/ -- url: https://opinion.cctv.com/ -- url: https://sports.cctv.com/ +# - url: https://www.cctv.com/ +# - url: https://sports.cctv.com/ diff --git a/tools/live_bench/refine_all_results.py b/tools/live_bench/refine_all_results.py new file mode 100644 index 000000000..fcc5beb35 --- /dev/null +++ b/tools/live_bench/refine_all_results.py @@ -0,0 +1,36 @@ +from tqdm import tqdm +from datasets import Dataset, load_dataset +from live_bench.data_generator.question_finalizer import QuestionFinalizer + + +if __name__ == "__main__": + hf_data = load_dataset("lmms-lab/LiveBench", "2024-07", split="test") + finalizer = QuestionFinalizer() + + def load_results(): + for item in tqdm(hf_data): + try: + res = finalizer.finalize_question(question=item["question"], answer=item["answer"], criteria=item["criteria"], images=item["images"]) + final_answer = item.copy() + final_answer["question"] = res["question"] + final_answer["answer"] = res["answer"] + final_answer["criteria"] = res["criteria"] + print(item) + print(final_answer) + except Exception as e: + print(f"Error in {item['id']}: {e}") + final_answer = item + + yield final_answer + # break + + final_data = {} + for data in load_results(): + for item, value in data.items(): + if item not in final_data: + final_data[item] = [] + final_data[item].append(value) + # final_data = Dataset.from_generator(load_results) + final_data = Dataset.from_dict(final_data, features=hf_data.features) + final_data.save_to_disk("logs/2024-07-final") + final_data.push_to_hub("lmms-lab/LiveBench", "2024-07") diff --git a/tools/live_bench/script/modify.ipynb b/tools/live_bench/script/modify.ipynb new file mode 100644 index 000000000..22944f1e0 --- /dev/null +++ b/tools/live_bench/script/modify.ipynb @@ -0,0 +1,461 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading data: 100%|██████████| 4.62k/4.62k [00:01<00:00, 4.14kB/s]\n", + "Generating test split: 100%|██████████| 8/8 [00:00<00:00, 933.60 examples/s]\n" + ] + } + ], + "source": [ + "import datasets\n", + "\n", + "data = datasets.load_dataset(\"lmms-lab/LiveBenchResults\", \"2024-07\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "df = data[\"test\"].to_pandas()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_csv(\"2024-07.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(\"2024-07.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Model NameTotalBasic UnderstandingContextual AnalysisDeeper ImplicationsBroader ImplicationsFurther Insights
0gpt-4o-mini86.24000089.081.089.687.60000084.0
1gemini-1.5-flash80.76000084.469.281.790.30000078.2
2gpt-4o91.30000092.283.494.194.40000092.4
3gemini-1.5-pro86.28000090.676.685.691.60000087.0
4llama3-llava-next-8b62.65060260.161.474.863.67346953.3
5llava-1.5-7b41.94000038.634.558.842.80000035.0
6Idefics2-8B25.86000018.016.343.827.00000024.2
7InternVL2-2B56.84000065.849.964.255.80000048.5
\n", + "
" + ], + "text/plain": [ + " Model Name Total Basic Understanding Contextual Analysis \\\n", + "0 gpt-4o-mini 86.240000 89.0 81.0 \n", + "1 gemini-1.5-flash 80.760000 84.4 69.2 \n", + "2 gpt-4o 91.300000 92.2 83.4 \n", + "3 gemini-1.5-pro 86.280000 90.6 76.6 \n", + "4 llama3-llava-next-8b 62.650602 60.1 61.4 \n", + "5 llava-1.5-7b 41.940000 38.6 34.5 \n", + "6 Idefics2-8B 25.860000 18.0 16.3 \n", + "7 InternVL2-2B 56.840000 65.8 49.9 \n", + "\n", + " Deeper Implications Broader Implications Further Insights \n", + "0 89.6 87.600000 84.0 \n", + "1 81.7 90.300000 78.2 \n", + "2 94.1 94.400000 92.4 \n", + "3 85.6 91.600000 87.0 \n", + "4 74.8 63.673469 53.3 \n", + "5 58.8 42.800000 35.0 \n", + "6 43.8 27.000000 24.2 \n", + "7 64.2 55.800000 48.5 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "data = datasets.Dataset.from_pandas(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Creating parquet from Arrow format: 100%|██████████| 1/1 [00:00<00:00, 473.45ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.17s/it]\n" + ] + }, + { + "data": { + "text/plain": [ + "CommitInfo(commit_url='https://huggingface.co/datasets/lmms-lab/LiveBenchResults/commit/a29f8ecb399dbd7ab7475f0de2c48ee54affbff9', commit_message='Upload dataset', commit_description='', oid='a29f8ecb399dbd7ab7475f0de2c48ee54affbff9', pr_url=None, pr_revision=None, pr_num=None)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.push_to_hub(\"lmms-lab/LiveBenchResults\", \"2024-07\", split=\"test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "data = datasets.load_dataset(\"lmms-lab/LiveBenchDetailedResults\", \"2024-07\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "data[\"Idefics2_8B\"] = data[\"idefics2\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " gpt_4o_mini: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " gemini_1.5_flash: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " gpt_4o: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " gemini_1.5_pro: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " llama3_llava_next_8b: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " llava_1.5_7b: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " idefics2: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " InternVL2_2B: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " Idefics2_8B: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + "})" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "new_data = {}\n", + "for k, v in data.items():\n", + " if k == \"idefics2\":\n", + " continue\n", + " new_data[k] = v\n", + "data = datasets.DatasetDict(new_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " gpt_4o_mini: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " gemini_1.5_flash: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " gpt_4o: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " gemini_1.5_pro: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " llama3_llava_next_8b: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " llava_1.5_7b: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " InternVL2_2B: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + " Idefics2_8B: Dataset({\n", + " features: ['id', 'images', 'question', 'ground_truth', 'criteria', 'subtask', 'response', 'score', 'reason'],\n", + " num_rows: 250\n", + " })\n", + "})" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Map: 100%|██████████| 250/250 [00:00<00:00, 347.35 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 12.58ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.63s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 363.40 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 12.70ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.59s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 472.60 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 12.62ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.43s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 352.11 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 11.55ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.63s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 475.90 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 11.38ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.46s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 364.89 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 10.94ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.59s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 529.96 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 13.51ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.33s/it]\n", + "Map: 100%|██████████| 250/250 [00:00<00:00, 349.67 examples/s]it/s]\n", + "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 12.74ba/s]\n", + "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00, 1.57s/it]\n" + ] + }, + { + "data": { + "text/plain": [ + "CommitInfo(commit_url='https://huggingface.co/datasets/lmms-lab/LiveBenchDetailedResults/commit/047d6dc66759e0a8b57b4e6015db6208da1cd4da', commit_message='Upload dataset', commit_description='', oid='047d6dc66759e0a8b57b4e6015db6208da1cd4da', pr_url=None, pr_revision=None, pr_num=None)" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.push_to_hub(\"lmms-lab/LiveBenchDetailedResults\", \"2024-07\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "live_bench", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tools/live_bench/script/upload_results.py b/tools/live_bench/script/upload_results.py index 9b7e45bf8..b8e608ffd 100644 --- a/tools/live_bench/script/upload_results.py +++ b/tools/live_bench/script/upload_results.py @@ -21,7 +21,7 @@ "criteria": datasets.Value("string"), "subtask": datasets.Value("string"), "response": datasets.Value("string"), - "score": datasets.Value("int32"), + "score": datasets.Value("float32"), "reason": datasets.Value("string"), } @@ -35,9 +35,7 @@ def load_images(config) -> Dict[int, List[Image.Image]]: - dataset = datasets.load_dataset( - config["dataset_path"], config["dataset_name"], split=config["test_split"] - ) + dataset = datasets.load_dataset(config["dataset_path"], config["dataset_name"], split=config["test_split"]) images = {} for data in tqdm(dataset, desc="Loading images"): images[data["id"]] = data["images"] @@ -46,24 +44,24 @@ def load_images(config) -> Dict[int, List[Image.Image]]: def get_hf_results(results, detailed_results): live_bench_images = load_images(results["configs"]["live_bench"]) - mapping = {k: [] for k in RESULT_FEATURES.keys()} - for result in tqdm(detailed_results["logs"], desc="Loading results"): - doc = result["doc"] - res = {} - res["id"] = doc["id"] - res["images"] = live_bench_images[doc["id"]] - res["question"] = doc["question"] - res["ground_truth"] = doc["answer"] - res["criteria"] = doc["criteria"] - res["subtask"] = doc["subtask"] - res["response"] = result["filtered_resps"][0] - res["score"] = result["gpt4_eval_score"]["rating"] - res["reason"] = result["gpt4_eval_score"]["explanation"] - for k, v in res.items(): - mapping[k].append(v) - result_dataset = datasets.Dataset.from_dict( - mapping=mapping, features=datasets.Features(RESULT_FEATURES) - ) + + def load_results(): + for result in tqdm(detailed_results["logs"], desc="Loading results"): + doc = result["doc"] + res = {} + res["id"] = doc["id"] + res["images"] = live_bench_images[doc["id"]] + res["question"] = doc["question"] + res["ground_truth"] = doc["answer"] + res["criteria"] = doc["criteria"] + res["subtask"] = doc["subtask"] + res["response"] = result["filtered_resps"][0] + res["score"] = result["gpt4_eval_score"]["rating"] + res["reason"] = result["gpt4_eval_score"]["explanation"] + yield res + + result_dataset = Dataset.from_generator(load_results, features=datasets.Features(RESULT_FEATURES)) + return result_dataset @@ -82,9 +80,7 @@ def calculate_score(results: Dataset): score = {} for subtask in SUBTASKS: score[subtask] = [] - for index, result in tqdm( - results.iterrows(), total=len(results), desc="Calculating score" - ): + for index, result in tqdm(results.iterrows(), total=len(results), desc="Calculating score"): if result["score"] == -1: continue sum_score += result["score"] / 10 @@ -93,10 +89,7 @@ def calculate_score(results: Dataset): if subtask not in SUBTASKS: subtask = "Further Insights" score[result["subtask"]].append(result["score"] / 10) - res = [ - (subtask, len(score[subtask]), np.mean(score[subtask]) * 100) - for subtask in SUBTASKS - ] + res = [(subtask, len(score[subtask]), np.mean(score[subtask]) * 100) for subtask in SUBTASKS] res.append(("Total", count, sum_score / count * 100)) res = pd.DataFrame(res, columns=["Subtask", "Count", "Score"]) @@ -110,16 +103,12 @@ def get_results(folder): with open(results_file, "r") as f: results = json.load(f) - assert ( - "live_bench" in results["configs"] - ), "No live_bench config found in results.json" + assert "live_bench" in results["configs"], "No live_bench config found in results.json" final_score = results["results"]["live_bench"]["gpt4_eval_score,none"] model_configs = results["model_configs"] version = results["configs"]["live_bench"]["metadata"]["version"] - assert ( - model_configs["limit"] is None - ), "Model limit is not None, please check if the model is tested on the full dataset" + assert model_configs["limit"] is None, "Model limit is not None, please check if the model is tested on the full dataset" with open(detailed_file, "r") as f: detailed_results = json.load(f) @@ -131,7 +120,7 @@ def get_results(folder): assert ( abs(score[score["Subtask"] == "Total"]["Score"] - final_score) <= EPS - ).all(), "Final score does not match the calculated score" + ).all(), f"Final score does not match the calculated score, the score calculated by the script is {score[score['Subtask'] == 'Total']['Score'].values[0]} and the final score in the log is {final_score}." return hf_results, score, version @@ -150,21 +139,15 @@ def upload_results( ) if not os.path.exists(log_folder): os.makedirs(log_folder) - score_path = os.path.abspath( - os.path.join(log_folder, f"{dataset_version}_{model_name}.csv") - ) + score_path = os.path.abspath(os.path.join(log_folder, f"{dataset_version}_{model_name}.csv")) score.to_csv(score_path, index=False) print(f"Results saved to {score_path}") score_dict = {item["Subtask"]: item["Score"] for index, item in score.iterrows()} score_dict["Model Name"] = model_name try: - hf_score = datasets.load_dataset( - "lmms-lab/LiveBenchResults", dataset_version, split="test" - ) + hf_score = datasets.load_dataset("lmms-lab/LiveBenchResults", dataset_version, split="test") except: - hf_score = Dataset.from_dict( - {subtask: [] for subtask in ["Model Name", "Total"] + SUBTASKS} - ) + hf_score = Dataset.from_dict({subtask: [] for subtask in ["Model Name", "Total"] + SUBTASKS}) hf_score = hf_score.add_item(score_dict) df_score = pd.DataFrame(hf_score) df_score = df_score.drop_duplicates(subset=["Model Name"], keep="last") @@ -175,19 +158,13 @@ def upload_results( if __name__ == "__main__": argparse = argparse.ArgumentParser() - argparse.add_argument( - "--folder", "-f", type=str, required=True, help="Results folder" - ) + argparse.add_argument("--folder", "-f", type=str, required=True, help="Results folder") argparse.add_argument("--name", "-m", type=str, required=True, help="Model name") - argparse.add_argument( - "--log_folder", "-l", type=str, default="logs", help="Log folder" - ) + argparse.add_argument("--log_folder", "-l", type=str, default="logs", help="Log folder") argparse.add_argument("--force", "-F", action="store_true", help="Force upload") args = argparse.parse_args() hf_results, score, version = get_results(args.folder) - print( - f"Results will be uploaded with model name {args.name} and model version {version}" - ) + print(f"Results will be uploaded with model name {args.name} and model version {version}") if args.force is False: print("Are you sure you want to upload the results? (y/n)", end=" ") while True: From d17027bfe7e01cd587b90065b4ab61e3f9277838 Mon Sep 17 00:00:00 2001 From: Kaichen Zhang - NTU Date: Fri, 26 Jul 2024 22:20:08 +0800 Subject: [PATCH 08/36] Add xcomposer2d5 from fanyi, revise something for better usage (#145) * internvl2 * fix some bugs * fix * lint * feat: Add XComposer2D5 model to AVAILABLE_MODELS * xcomposer * Fix llava vid error when using public * Fix xcomposer2d5 * Add generation tokens --------- Co-authored-by: Fanyi Pu --- lmms_eval/models/__init__.py | 1 + lmms_eval/models/internvl2.py | 36 +++++- lmms_eval/models/llava_vid.py | 13 ++- lmms_eval/models/xcomposer2d5.py | 183 +++++++++++++++++++++++++++++++ lmms_eval/tasks/mlvu/mlvu.yaml | 6 + 5 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 lmms_eval/models/xcomposer2d5.py diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py index b618f8a35..7ba7264c5 100755 --- a/lmms_eval/models/__init__.py +++ b/lmms_eval/models/__init__.py @@ -39,6 +39,7 @@ "llava_onevision": "LlavaOneVision", "longva": "LongVA", "vila": "VILA", + "xcomposer2d5": "XComposer2D5", } for model_name, model_class in AVAILABLE_MODELS.items(): diff --git a/lmms_eval/models/internvl2.py b/lmms_eval/models/internvl2.py index 5843bdd10..4fdad818e 100644 --- a/lmms_eval/models/internvl2.py +++ b/lmms_eval/models/internvl2.py @@ -187,9 +187,41 @@ def __init__( self._rank = 0 self._world_size = 1 - self.device = self._device self.modality = modality + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def tokenizer(self): + return self._tokenizer + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + def flatten(self, input): new_list = [] for i in input: @@ -204,7 +236,6 @@ def generate_until(self, requests) -> List[str]: for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: if "until" in gen_kwargs: gen_kwargs.pop("until") - for k, v in DEFAULT_GEN_KWARGS.items(): if k not in gen_kwargs: gen_kwargs[k] = v @@ -228,7 +259,6 @@ def generate_until(self, requests) -> List[str]: image_tokens = " ".join(image_tokens) contexts = image_tokens + "\n" + contexts response, history = self.model.chat(self.tokenizer, pixel_values, contexts, gen_kwargs, num_patches_list=num_patches_list, history=None, return_history=True) - elif self.modality == "video": assert len(visuals) == 1, f"Only one video is supported, but got {len(visuals)} videos." video_path = visuals[0] diff --git a/lmms_eval/models/llava_vid.py b/lmms_eval/models/llava_vid.py index 14cd7e610..27da7a949 100755 --- a/lmms_eval/models/llava_vid.py +++ b/lmms_eval/models/llava_vid.py @@ -26,10 +26,13 @@ except ImportError: eval_logger.debug("LLaVA-Video is not installed. Please install LLaVA-Video to use this model.") -from llavavid.model.language_model.llava_qwen import LlavaQwenConfig -from llavavid.model.language_model.llava_llama import LlavaConfig +try: + from llavavid.model.language_model.llava_qwen import LlavaQwenConfig + AutoConfig.register("llava_qwen", LlavaQwenConfig) +except ImportError: + eval_logger.debug("No Qwen for llava vid") -AutoConfig.register("llava_qwen", LlavaQwenConfig) +from llavavid.model.language_model.llava_llama import LlavaConfig AutoConfig.register("llava_llama", LlavaConfig) @@ -57,6 +60,8 @@ def __init__( mm_spatial_pool_stride: int = 2, mm_spatial_pool_out_channels: int = 1024, mm_spatial_pool_mode: str = "average", + mm_newline_position: str = "grid", + mm_pooling_position: str = "after", overwrite: bool = True, video_decode_backend: str = "pyav", delay_load: bool = False, @@ -88,7 +93,7 @@ def __init__( self.mm_spatial_pool_out_channels = int(mm_spatial_pool_out_channels) self.mm_spatial_pool_mode = mm_spatial_pool_mode self.max_frames_num = int(max_frames_num) - self.mm_resampler_location = mm_resampler_location + self.mm_resampler_location = mm_pooling_position self.delay_load = delay_load if self.overwrite == True: overwrite_config = {} diff --git a/lmms_eval/models/xcomposer2d5.py b/lmms_eval/models/xcomposer2d5.py new file mode 100644 index 000000000..17ed8fa25 --- /dev/null +++ b/lmms_eval/models/xcomposer2d5.py @@ -0,0 +1,183 @@ +from typing import List, Tuple +from lmms_eval.api.instance import Instance +from decord import VideoReader, cpu +import torch +import torchvision.transforms as T +from PIL import Image +from torchvision.transforms.functional import InterpolationMode +import numpy as np +from transformers import AutoModel, AutoTokenizer +from lmms_eval.api.registry import register_model +from accelerate import Accelerator, DistributedType +from lmms_eval.api.model import lmms +from tqdm import tqdm +import logging +import os + +eval_logger = logging.getLogger("eval_logger") + + +from datetime import timedelta +from accelerate.state import AcceleratorState +from accelerate.utils import InitProcessGroupKwargs + + +@register_model("xcomposer2d5") +class XComposer2D5(lmms): + def __init__( + self, + pretrained: str = "internlm/internlm-xcomposer2d5-7b", + modality: str = "image", + device: str = "cuda:0", + device_map: str = "cuda:0", + batch_size: str = "1", + tmp_folder: str = "./temp/xcomposer2d5/", + **kwargs, + ): + super().__init__() + + self.tmp_folder = os.path.abspath(tmp_folder) + if not os.path.exists(self.tmp_folder): + os.makedirs(self.tmp_folder) + eval_logger.info(f"Using temporary folder: {self.tmp_folder}") + + + batch_size = int(batch_size) + assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}." + self.batch_size_per_gpu = batch_size + + accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52)) + accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs]) + if accelerator.num_processes > 1: + self._device = torch.device(f"cuda:{accelerator.local_process_index}") + self.device_map = f"cuda:{accelerator.local_process_index}" + elif accelerator.num_processes == 1 and device_map == "auto": + self._device = torch.device(device) + self.device_map = device_map + else: + self._device = torch.device(f"cuda:{accelerator.local_process_index}") + self.device_map = f"cuda:{accelerator.local_process_index}" + + self.path = pretrained + self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map=self.device_map).half().eval() + self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True) + self._model.tokenizer = self._tokenizer + + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model + # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works + # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work. + if accelerator.distributed_type == DistributedType.DEEPSPEED: + kwargs = { + "train_micro_batch_size_per_gpu": self.batch_size_per_gpu, + "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes, + } + AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs) + eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0") + + if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED: + self._model = accelerator.prepare(self.model) + else: + self._model = accelerator.prepare_model(self.model, evaluation_mode=True) + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + elif accelerator.num_processes == 1 and device_map == "auto": + eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism") + self._rank = 0 + self._word_size = 1 + else: + eval_logger.info(f"Using single device: {self._device}") + self.model.to(self._device) + self._rank = 0 + self._world_size = 1 + + self.modality = modality + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def tokenizer(self): + return self._tokenizer + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def generate_until(self, requests) -> List[str]: + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + image = [] + + if self.modality == "image": + for idx, visual in enumerate(visuals): + visual.save(os.path.join(self.tmp_folder, f"tmp_{idx}_{self.rank}_{self.world_size}.jpg")) + image.append(os.path.join(self.tmp_folder, f"tmp_{idx}_{self.rank}_{self.world_size}.jpg")) + if image: + image_tokens = [f"Image{i} ; " for i in range(len(visuals))] + image_tokens = "".join(image_tokens) + contexts = image_tokens + contexts + elif self.modality == "video": + image = visuals + + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 1024 + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + if "top_p" not in gen_kwargs: + gen_kwargs["top_p"] = None + if "num_beams" not in gen_kwargs: + gen_kwargs["num_beams"] = 1 + + try: + with torch.autocast(device_type='cuda', dtype=torch.float16): + response, his = self.model.chat(self.tokenizer, contexts, image, do_sample=False, num_beams=1, use_meta=True, max_new_tokens=gen_kwargs["max_new_tokens"]) + except Exception as e: + eval_logger.error(f"Error : {e}") + response = "" + + res.append(response) + pbar.update(1) + pbar.close() + return res + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + assert False, "Not implemented yet." diff --git a/lmms_eval/tasks/mlvu/mlvu.yaml b/lmms_eval/tasks/mlvu/mlvu.yaml index 375da5958..b76db88c1 100644 --- a/lmms_eval/tasks/mlvu/mlvu.yaml +++ b/lmms_eval/tasks/mlvu/mlvu.yaml @@ -3,6 +3,12 @@ dataset_kwargs: token: True cache_dir: mlvu video: True +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false task: mlvu test_split: test output_type: generate_until From 9dd1672f8a264dd2057ae51fafc214ff21f72100 Mon Sep 17 00:00:00 2001 From: Li Bo Date: Sat, 27 Jul 2024 13:43:24 +1000 Subject: [PATCH 09/36] Dev/ov evals (#147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix doc * [WIP] adding mmbench dev evaluation (#75) * WIP * Update GPT evaluation model name and sys prompt * 🛠️ Scale accuracy to percentage The accuracy value is now multiplied by 100 in the aggregation function to represent it as a percentage. Regarding the evaluation process, `math` module importation and refactoring reduce progress log verbosity by logging every 100 evaluations instead of 10. It prevents potential logging overflow. Handling of NaN values is added to ensure 'default_value' is set in case of missing data, avoiding errors in split, category, and l2-category assignments. Finally, reporting of categorical and l2-categorical accuracies is streamlined through a new `calculate_hit_rates` function, improving code readability and maintenance. Issue refs: #1427, #1533 * Update GPT evaluation model name and API configuration * Refactor MMBench_Evaluator class to handle missing columns * Add print statements for detailed results in MMBench-CN(CC), MMBench-CN(Dev), and MMBench-EN(Dev) evaluations * Refactor MMBench-CN and MMBench-EN evaluation functions * 🔄 Refactor result processing and logging logic - Simplified the result processing functions across different utility modules (`cc_utils.py`, `cn_utils.py`, `en_utils.py`) to unify the handling of multiple-choice options. Now, all options ("A" to "E") are dynamically added to the result data, and default to "nan" if not provided in the document. - Removed redundant keys directly from the process results dict creation to avoid clutter and align with the new dynamic addition of options. - In `mmbench_evals.py`, removed the unnecessary check for all splits being 'dev' and streamlined the evaluation loop by eliminating the progress bar (tqdm) for a cleaner log output. - Commented-out code and verbose logging during evaluation, which may have interfered with performance, has been removed for a more efficient and less intrusive logging experience. This cleanup reduces redundancy in the codebase and improves evaluation performance. Refs #2045 --------- Co-authored-by: Bo Li (cherry picked from commit a19278c2ea6ddcbca64d3cc7f4efec7fe5775121) * Create README.md * Add files via upload * Add MathVerse * Fix typo in qwen_vl that was causing "reference before assignment" * convert contexts to list if necessary and remove unnecessary construction of `questions` * refactor query construction for clarity * Create ScreenSpot on clean branch * Update README to reflect new tasks * Add README file specific to ScreenSpot * slight update * Init webSRC * Draft README for WebSRC * Update main README with new task names * Draft and validate websrc eval on dev split * Add code to enable compilation of submission for WebSRC test split * Bugfix: WebSRC should be token-level F1 NOT character-level * Add qwen vl api * Fix llava conv template for llama3 * Fix llava_hf generation for 1.6 * Parse result for llava_hf 1.6 * Add model_name parameter to Llava constructor * Fix endless warning for llava_hf generation * Fix llava_hf image tokens number issue * Create LICENSE * Update LICENSE * Update LICENSE * Better task list_with_num * Fix idefics2 llava in the wild bugs * Remove redundant code in fuyu * Fix instructblip qformer size mismatch and multi-images problem * Comment out parse result in xcomposer * Comment out Spice in caption task so that don't need to download stanford nlp model * Update gitignore * Add separated pope tasks by category * Fix pope random name in pope full * Set printing info for llava_hf to debug level * Adding Phi3v model. * Adding prompt arguments for Phi3v on MathVista-TestMini * Adding documentation of Phi3v class. * [Fix] import issues of multilingual llava and olympiadbench * fix compatibility issue of older version llava * add upd * add upd * add upd * add upd * add upd * add upd * Group MMMU images into one image (#83) * update * update font * Add matplotlib.font_manager import in utils.py * Refactor font handling in add_order_label function in utils.py * group mmmu --------- Co-authored-by: Li Bo * merge model_specific_prompt_kwargs and dataset_name into each task yaml * Add MathVerse in README.md * slightly change query_prompt for the reproduction * update utils.py for leaderboard submission * add conbench * update README * Update README.md * init include vcr * modify the form of VCR * switch logic * add crossed_text to vcr_wiki output * include the try-except logic for spacy * update vcr_wiki tasks * update vcr_wiki tasks in README.md * include std and confidence interval * update gpt-3.5-turbo version * update gpt-3.5-turbo version * chore: Remove unnecessary files and code related to live_bench and sft_eval tasks * Bump version to 0.2.0.dev0 * chore: Update lmms-eval to support video evaluations for LLaVA models * Update llava conv_template in lmms_eval/models/llava.py * Update image alignment in README.md * chore: Update lmms-eval to support video evaluations for LLaVA models * chore: Update lmms-eval to support video evaluations for LLaVA models * Update README.md * Update README.md * update aggregation function for vcr_wiki * update README.md * Update README.md * update version * add II-Bench * fix dataset_path * Add qbench, qbench2, abench; fix phi3v as its current implementation does not support multi-image * add tinyllava * LongVideoBench support: image LMMs (idefics2, phi3) and video LMMs (LLaVA-Next-Video-34B) * fix #117, allow auto download with tar format videos * fix #117, allow auto download with tar format videos * fix typo * feat: Add support for auto downloading tar format videos * Release llava-wilder * chore: Update dependencies to fix potential risks and improve compatibility * tutorial * docs * update preparation * small fix * small fix * lint * to sh script * update readme * Remove handling non-visual loop in llava * Add llava_hf back to registry * Update README.md * Update README.md * update ablation for videomme datasets * chore: Handle ImportError when importing models Handle the ImportError exception when importing models in the lmms_eval package. This change adds a try-except block to catch the ImportError and print an error message indicating the failed import. This will help with troubleshooting and identifying any issues with the model imports. * chore: Remove unused models from lmms_eval package * feat: Allow loading model configurations from other packages * feat: Allow including external tasks from plugins * chore: Add loguru for logging in lmms_eval package * Remove unnecessary lines since use batched visuals now in llava * Add longva * Revise model registry for llava_hf and longva * Delete unnecessary lines * Remove unnecessary lines for video llava * Update pyproject.toml * Update activitynetqa_generation.yaml * Fix vid mme post prompt issue * new task gqa-ru * add mmbench_ru_dev * change prompt to ru * create new task vitatecs * Update README.md * Add wild vision 0617 * Hardcode to keep image for wild vision * Fixing scoring logic * Fixing dataset name * Fixing handling None filtered score * Add detailcaps * Add install capture_metric in env * Add files via upload * feat: Add tie_weights parameter to Llava model initialization * Upgrade lmms-eval to support more models and evaluation tasks * Upgrade lmms-eval to version 0.2.1 * Rename xcomposer 4KHD * chore: Update lmms_eval/models/vila.py and lmms_eval/tasks/__init__.py * Update utils.py * Update _default_template_vcr_yaml * add process sync via temp file in lmms_eval/evaluator.py * Update utils.py * Update _default_template_vcr_yaml * Add muirbench * Squashed commit of the following: commit dfdba507b5fbe985b0030ffec575f9f2638bc1ed Author: Li Bo Date: Tue Jul 16 11:13:52 2024 +0800 merge ov evals (#144) * chore: Update gpt_eval_model_name to "gpt-3.5-turbo" in mathvista.yaml * Squashed commit of the following: commit 994c9f97a2f8db3e9b7d7933d1e1680acde5b70b Author: Yan Shu <570533048@qq.com> Date: Mon Jul 8 17:21:23 2024 +0800 Add files via upload * Squashed commit of the following: commit e31cd7883d4555c7530795c7f102b8d78cbd372f Author: Bo Li Date: Wed Jul 10 12:08:08 2024 +1000 chore: Update lmms_eval/models/vila.py and lmms_eval/tasks/__init__.py commit 1d8c980d1089f9d7702c3b92d5c85039f2809c6d Author: kcz358 Date: Tue Jul 9 02:08:52 2024 +0000 Rename xcomposer 4KHD commit 6da76f36ecf5f9aa73057e767a4fcb60c99ff896 Author: Bo Li Date: Tue Jul 9 11:55:56 2024 +1000 Upgrade lmms-eval to version 0.2.1 commit cd1858523fcd8630082cbefba8710e0de3ee8805 Author: Bo Li Date: Tue Jul 9 11:52:23 2024 +1000 Upgrade lmms-eval to support more models and evaluation tasks commit 672d7e5bb49dcb34e1b2fdeb09f3f4588dc583a6 Author: Bo Li Date: Tue Jul 9 11:43:41 2024 +1000 feat: Add tie_weights parameter to Llava model initialization commit 2037a86261b55fa42b8ba3a04eab192b3e69d6ea Merge: e6844db1 a5c18692 Author: Bo Li Date: Tue Jul 9 11:37:12 2024 +1000 Fix gen kwargs image aspect ratio in internvl2 commit a5c186925de989b616f58a35ece36065a32b4594 Merge: 2ebec77f 557083a1 Author: Li Bo Date: Tue Jul 9 09:15:56 2024 +0800 Merge pull request #137 from shuyansy/main add MLVU task commit 557083a156c3dd67ac79e22b4202e9b69b6b00f4 Author: Yan Shu <570533048@qq.com> Date: Mon Jul 8 16:56:50 2024 +0800 Add files via upload commit 2ebec77f5606d79e9a7b995970e32792050606a1 Merge: 211bfede b23d349e Author: Li Bo Date: Mon Jul 8 11:53:06 2024 +0800 Merge pull request #136 from Dousia/main Add detailcaps commit b23d349e46d60dc149ffaa54d6e019f4996ed92d Author: ByteDance Date: Sun Jul 7 23:24:19 2024 +0800 Add install capture_metric in env commit c6e211d5f9dbb7572d3a141b6504cb1ca2007c33 Author: ByteDance Date: Sun Jul 7 23:04:13 2024 +0800 Add detailcaps commit 211bfedebad243ef82a8b0be36c3b5a9b9cb2f72 Merge: 7c208b76 79514eee Author: Li Bo Date: Tue Jul 2 23:05:12 2024 +0800 Merge pull request #133 from EvolvingLMMs-Lab/dev/wild_vision Add wild vision bench commit 79514eeebcfd6f655be2a10c776037d12a7b7214 Author: kcz358 Date: Mon Jul 1 15:10:02 2024 +0000 Fixing handling None filtered score commit 725fac2781446958b905e1e6c6eb3c0a8e582e49 Author: kcz358 Date: Mon Jul 1 08:25:42 2024 +0000 Fixing dataset name commit 8d963e132ac03fc0d835d480cfcfcabe72af143c Author: kcz358 Date: Mon Jul 1 08:24:51 2024 +0000 Fixing scoring logic commit e2990d0a69e876721256fdf946c68ba7ae0cbdc1 Author: kcz358 Date: Mon Jul 1 06:06:57 2024 +0000 Hardcode to keep image for wild vision commit ed381736730d8fb785b4ee919fdb751734ecef25 Author: kcz358 Date: Mon Jul 1 06:06:38 2024 +0000 Add wild vision 0617 commit 7c208b76640c986cfe94233dce735c3ca4ad4319 Author: Li Bo Date: Mon Jul 1 11:53:31 2024 +0800 Update README.md commit 39d40dea47bc59ff04e8b0cbc445345098debc9a Merge: e19b43a3 ba7081c0 Author: Li Bo Date: Mon Jul 1 11:47:09 2024 +0800 Merge pull request #129 from Dannoopsy/mmbench_ru add task MMBench-ru commit e19b43a3a1e7212e623061b164b0419cc0dda689 Merge: 11fd7e3f a0de8970 Author: Li Bo Date: Mon Jul 1 11:46:58 2024 +0800 Merge pull request #128 from Dannoopsy/gqa-ru add task gqa-ru commit 11fd7e3fc05908aeb01e4a6161a7b55cd38b3122 Merge: 383e7fea a7522592 Author: Li Bo Date: Mon Jul 1 11:46:16 2024 +0800 Merge pull request #130 from lscpku/vitatecs Add task VITATECS commit a75225926e5954f85466d257f99acf0163fde596 Author: lscpku Date: Fri Jun 28 20:37:06 2024 +0800 create new task vitatecs commit ba7081c0abac840002d320e30733e891298dfa11 Author: Dannoopsy <63581325+Dannoopsy@users.noreply.github.com> Date: Fri Jun 28 12:21:05 2024 +0300 change prompt to ru commit 27ea9c0055a8abf3a8198829b8617018479918e2 Author: Dannoopsy Date: Thu Jun 27 17:17:29 2024 +0000 add mmbench_ru_dev commit 383e7fead3138aedf62e9c0ec48303835ef26e2a Merge: 06fa000f ed2e7f79 Author: Li Bo Date: Fri Jun 28 00:14:10 2024 +0800 Merge pull request #126 from lorenzomammana/feature/external-package-integration External package integration using plugins commit ed2e7f792151d21bce8f1c498270b9391e1d5c85 Merge: 03947e14 06fa000f Author: Lorenzo Mammana Date: Thu Jun 27 15:38:10 2024 +0000 Merge branch 'main' into feature/external-package-integration commit a0de89708d5e6f259bb17f0eaace3c5b901b275c Author: Dannoopsy Date: Tue Jun 25 11:11:37 2024 +0000 new task gqa-ru commit 06fa000f60d3e4d160fac8ceb9959ae92a98f752 Author: kcz358 Date: Tue Jun 25 06:41:13 2024 +0000 Fix vid mme post prompt issue commit b388d79e0df6f60068196cb7047453ebd22d6ef1 Author: Li Bo Date: Sun Jun 23 22:31:16 2024 +0800 Update activitynetqa_generation.yaml commit 8f9d620fcd9d0a0742ee6bcf51ea63bd6b088a36 Author: Li Bo Date: Sun Jun 23 14:02:25 2024 +0800 Update pyproject.toml commit 6341b7c15ce9fb28eb06b067ddb299d6cf2e16c3 Merge: fce85f1b 903b042b Author: Li Bo Date: Sun Jun 23 14:02:02 2024 +0800 Merge pull request #125 from EvolvingLMMs-Lab/dev/interleave [Model] aligned llava-interleave model results on video tasks commit 903b042be016016d4ebeecb07701f3076a2d323c Author: kcz358 Date: Sat Jun 22 12:07:13 2024 +0000 Remove unnecessary lines for video llava commit d78ec86407b729a964906a8c2e50704b4bc74d06 Merge: ebe7217a fce85f1b Author: Li Bo Date: Sat Jun 22 13:57:31 2024 +0800 Merge branch 'main' into dev/interleave commit ebe7217a486c1e754e42c2cbdb834e09fbbcc9b0 Author: kcz358 Date: Sat Jun 22 02:57:08 2024 +0000 Delete unnecessary lines commit 120c474b056f9177c74e1fd9691d59e2f234b785 Author: kcz358 Date: Fri Jun 21 08:38:41 2024 +0000 Revise model registry for llava_hf and longva commit 7d6201f921088afd3f52a35076e3c6fcc9aa518c Author: kcz358 Date: Fri Jun 21 08:38:24 2024 +0000 Add longva commit 12f480699c71a12a24d4349d9b0681933201a3a6 Author: kcz358 Date: Fri Jun 21 08:35:39 2024 +0000 Remove unnecessary lines since use batched visuals now in llava commit 12cea76f1f0f14b1fd1007c9d39a9b0557368637 Author: Bo Li Date: Thu Jun 20 18:15:32 2024 +0000 chore: Add loguru for logging in lmms_eval package commit 03947e14a46fd25b412931f7c9c25f4a2971d0b4 Author: Lorenzo Mammana Date: Wed Jun 5 13:40:41 2024 +0000 feat: Allow including external tasks from plugins commit b80a91f73e15ddd0b0ce1322d7d121fa14030eed Author: Lorenzo Mammana Date: Wed Jun 5 13:04:55 2024 +0000 feat: Allow loading model configurations from other packages commit 8ef24740dd48a11c97eb627f2fff4aca107fef0d Author: Bo Li Date: Thu Jun 20 12:11:03 2024 +0000 chore: Remove unused models from lmms_eval package commit af38885fc2e066f5ea44388f33e07176f836fe28 Author: Bo Li Date: Thu Jun 20 12:07:09 2024 +0000 chore: Handle ImportError when importing models Handle the ImportError exception when importing models in the lmms_eval package. This change adds a try-except block to catch the ImportError and print an error message indicating the failed import. This will help with troubleshooting and identifying any issues with the model imports. commit fce85f1b03ff7043b29dee787c5d17a08dd2687a Merge: dbe63293 d94f83cb Author: Li Bo Date: Thu Jun 20 20:02:12 2024 +0800 Merge pull request #120 from EvolvingLMMs-Lab/pufanyi/hf_dataset_docs Add docs for datasets upload to HF commit dbe63293245a5141fdfd80bda7657c304f6bd32f Author: choiszt Date: Thu Jun 20 15:14:21 2024 +0800 update ablation for videomme datasets commit d94f83cb3f08b61a2c75cc4326e58792100605b3 Author: Li Bo Date: Thu Jun 20 13:30:59 2024 +0800 Update README.md commit cab8159ff35db330536c0b6dfb4b0a3b24142209 Author: Li Bo Date: Thu Jun 20 13:30:29 2024 +0800 Update README.md commit 45876652a877a8006b828f32f5cc4660629f9190 Author: kcz358 Date: Thu Jun 20 03:55:30 2024 +0000 Add llava_hf back to registry commit 3463651b8c54d36cd94169e3d376f5ed225a195a Author: kcz358 Date: Thu Jun 20 03:54:33 2024 +0000 Remove handling non-visual loop in llava commit cb0d3f49b72790b081f981e0e6147131542f7f68 Author: Fanyi Pu Date: Thu Jun 20 02:11:18 2024 +0800 update readme commit 813877bfe5ac590cdbe92dd74d18f83a2091f748 Author: Fanyi Pu Date: Wed Jun 19 15:37:52 2024 +0800 to sh script commit a14684b8557d5894976448a5c559ed7a66a6cf16 Author: Fanyi Pu Date: Wed Jun 19 15:37:04 2024 +0800 lint commit d0f8851d42ba31f5da2a7a65e91499db45174dbc Author: Fanyi Pu Date: Wed Jun 19 15:36:48 2024 +0800 small fix commit 63748e9718f287ad433afc90e340b5e17a89c1ed Author: Fanyi Pu Date: Wed Jun 19 15:36:43 2024 +0800 small fix commit 7f1159a1fe04cfb783dc31d4fbdef3bda0ce19e4 Author: Fanyi Pu Date: Wed Jun 19 15:35:05 2024 +0800 update preparation commit 19f9bd621c76a483ff98f8c7eb78f64753da683a Author: Fanyi Pu Date: Wed Jun 19 15:23:24 2024 +0800 docs commit ce6f889ba02d819979c7922f6336cf4f1f718f65 Author: Fanyi Pu Date: Wed Jun 19 15:04:16 2024 +0800 tutorial commit f513c520c2a3dad26d2b2ca5c4ed4db05a493c73 Author: Bo Li Date: Wed Jun 19 06:51:19 2024 +0000 chore: Update dependencies to fix potential risks and improve compatibility commit efb529552c5e4ba039a4cba8e9aa5cb7ba65bf90 Author: kcz358 Date: Wed Jun 19 10:25:58 2024 +0800 Release llava-wilder commit 742651fc9daf97e2f57831ed6e6e7ee7ead7d555 Author: Fanyi Pu Date: Wed Jun 19 07:44:26 2024 +0800 feat: Add support for auto downloading tar format videos commit 511b6259828212fcba954cdeb8cf90d6e5daabf8 Merge: 22a4958e 050b2c37 Author: Bo Li Date: Tue Jun 18 17:01:03 2024 +0000 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval commit 050b2c370017e9b97475dd6cf01fd051b5ca5c86 Merge: 74facb41 ef306512 Author: Li Bo Date: Tue Jun 18 13:13:38 2024 +0800 Merge pull request #114 from zjysteven/add-tinyllava add tinyllava commit ef306512e5135f76dffa383f600b8733015836e8 Author: Jingyang Zhang Date: Mon Jun 17 17:57:02 2024 -0400 fix typo commit 9bab67732a4238097725deddf867fb1946ffee40 Merge: dbfb2387 74facb41 Author: Jingyang Zhang Date: Sun Jun 16 10:56:05 2024 -0400 Merge branch 'EvolvingLMMs-Lab:main' into add-tinyllava commit 74facb41a826691dfce4458cf1d8659b34fc5bf5 Merge: 8ba192f9 d5df72de Author: Li Bo Date: Sun Jun 16 17:59:19 2024 +0800 Merge pull request #118 from teowu/main Fix the potential risk by PR #117 commit d5df72de2d03108d6b365818ecc3551ac9aa6302 Merge: 5bf59ed2 8ba192f9 Author: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Date: Sun Jun 16 15:32:13 2024 +0800 Merge branch 'EvolvingLMMs-Lab:main' into main commit 5bf59ed250da98a408a94e214a73caa400cba842 Author: teowu Date: Sun Jun 16 07:27:28 2024 +0000 fix #117, allow auto download with tar format videos commit 98b3955cb808e36303c030aea78eb037d1ec59ce Merge: a056f118 be9dada8 Author: teowu Date: Sun Jun 16 07:25:07 2024 +0000 Merge branch 'main' of https://github.com/teowu/lmms-eval into main commit a056f118704eccec86ce32ab86981ce4bc1e1deb Author: teowu Date: Sun Jun 16 07:23:54 2024 +0000 fix #117, allow auto download with tar format videos commit 8ba192f94edf5d99598983445d5faa4f8807c49f Merge: 7cc28907 be9dada8 Author: Li Bo Date: Sat Jun 15 17:30:59 2024 +0800 Merge pull request #117 from teowu/main LongVideoBench for LMMs-Eval commit be9dada8b4189c53c08e1674ab273242cf2f80a0 Merge: 62ea8ceb 7cc28907 Author: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Date: Sat Jun 15 16:39:20 2024 +0800 Merge pull request #1 from EvolvingLMMs-Lab/main Merge pull request #113 from teowu/main commit 62ea8ceb223ef2b51ebab2bcd50d5cf339c35cfe Author: teowu Date: Sat Jun 15 08:30:11 2024 +0000 LongVideoBench support: image LMMs (idefics2, phi3) and video LMMs (LLaVA-Next-Video-34B) commit 7cc28907edbb4eb58ee1398772a48110ea35dd96 Merge: 4bc7224d ea14cd4b Author: Li Bo Date: Sat Jun 15 14:10:22 2024 +0800 Merge pull request #113 from teowu/main Q-Bench, Q-Bench2, A-Bench commit dbfb23873979f789477f4797ee2d6071e0fd921e Author: Jingyang Date: Fri Jun 14 16:20:42 2024 -0400 add tinyllava commit ea14cd4b361f4c95b3665cbdb95bc51754090eb5 Author: teowu Date: Fri Jun 14 15:01:52 2024 +0000 Add qbench, qbench2, abench; fix phi3v as its current implementation does not support multi-image commit 4bc7224dcd27fe8b288bfc3fed4d7a9da9635658 Merge: 2797987f bf14cb85 Author: Li Bo Date: Fri Jun 14 02:14:43 2024 +0800 Merge pull request #111 from XinrunDu/main add II-Bench commit bf14cb8527b2b7ac438a36567a875168bc02d294 Author: XinrunDu Date: Thu Jun 13 09:37:02 2024 +0000 fix dataset_path commit 6248113f4e11a0ac396d31fa1b032a142fea8cb4 Author: XinrunDu Date: Thu Jun 13 09:32:06 2024 +0000 add II-Bench commit 2797987f5b88b87bd172714b678a75a1d8051826 Merge: 63d82f1f 66d4bb2d Author: Li Bo Date: Thu Jun 13 11:14:47 2024 +0800 Merge pull request #109 from EvolvingLMMs-Lab/pufanyi/update_version [Small Update] Update the version of LMMs-Eval commit 66d4bb2d9c9afbbdea40196d4ad80e214d0b14b6 Author: Fanyi Pu Date: Thu Jun 13 11:13:00 2024 +0800 update version commit 63d82f1ff11eb430d91a15d6788a1f0b4d596850 Author: Li Bo Date: Thu Jun 13 11:04:32 2024 +0800 Update README.md commit 44a33799671cb668f55366d5e5a4ddb051a3a1b4 Merge: 5ed00356 0ce46d08 Author: Li Bo Date: Thu Jun 13 04:00:12 2024 +0800 Merge pull request #105 from tianyu-z/main Include VCR commit 0ce46d088e473d12d63de44f17c67dceab25658c Author: Suyuchen Date: Wed Jun 12 15:56:34 2024 -0400 update README.md commit 46a88d8b0199ed44d2ff459fb372f2e006960cea Merge: 47b13b9b 5ed00356 Author: Suyuchen Date: Wed Jun 12 15:50:26 2024 -0400 merged readme.md commit 47b13b9b320d36ac53b3622557e31239f7c22621 Author: Suyuchen Date: Wed Jun 12 15:30:52 2024 -0400 update aggregation function for vcr_wiki commit 5ed00356676cf5d0ff056cf27d1b519b8e303ff7 Author: Li Bo Date: Thu Jun 13 03:21:42 2024 +0800 Update README.md commit ed8806839db5988ced672bd162b7b046edb4863a Author: Li Bo Date: Thu Jun 13 03:13:59 2024 +0800 Update README.md commit fea3806026932a6e2bd6e538bcc413e33abdf245 Merge: d99a24ab 05dc8e85 Author: Li Bo Date: Thu Jun 13 03:11:49 2024 +0800 Merge pull request #108 from EvolvingLMMs-Lab/internal_main_dev [Upgrade to v0.2] Embracing Video Evaluations with LMMs-Eval commit 05dc8e853eab7c6bc782a1e2662d2efe7422f767 Author: Bo Li Date: Wed Jun 12 15:56:04 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit cbeee20bc4ffb510a2b23d96cdaf4077be7c2a9e Author: Bo Li Date: Wed Jun 12 15:50:30 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit f00d5498b69dd4f7e54c907ac906abc7c128f000 Author: Bo Li Date: Wed Jun 12 15:46:33 2024 +0000 Update image alignment in README.md commit 34156335db74cef9e3f0915d7172fd6b22456c15 Author: Bo Li Date: Wed Jun 12 15:43:16 2024 +0000 Update llava conv_template in lmms_eval/models/llava.py commit 50575a950736bc8fc1e191310314cbb5fdff5720 Author: Bo Li Date: Wed Jun 12 15:39:03 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit c9b2252fb8a15dd04252af5e6b4613855afd6ada Author: Bo Li Date: Wed Jun 12 15:33:48 2024 +0000 Bump version to 0.2.0.dev0 commit 465bd4205e8097e9c037b24a3ed08dd6a7694efa Merge: e43bd840 d99a24ab Author: Bo Li Date: Wed Jun 12 15:04:25 2024 +0000 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval into internal_main_dev commit e43bd840b63eb499856e36d9d2ba45c924abcead Author: Bo Li Date: Wed Jun 12 14:54:06 2024 +0000 chore: Remove unnecessary files and code related to live_bench and sft_eval tasks commit d99a24abd06df10d07e5a4d0ad5030613f92f2e7 Merge: 374590be a66003be Author: Li Bo Date: Wed Jun 12 19:45:57 2024 +0800 Merge pull request #107 from AtsuMiyai/new_task/upd_update update gpt-3.5-turbo version commit a66003befe4175824a1be6ed59f5f5b88c15f792 Author: AtsuMiyai Date: Wed Jun 12 17:05:17 2024 +0900 update gpt-3.5-turbo version commit ee91f272985f32eeb9cd6faa41afdd8eb49cac30 Author: AtsuMiyai Date: Wed Jun 12 16:50:53 2024 +0900 update gpt-3.5-turbo version commit 326b9694fc77398592b8caf3ba0bc2e2bb903813 Author: tianyu-z Date: Mon Jun 10 20:07:40 2024 -0400 include std and confidence interval commit cd050d4a721d01a2ace0cd030cf7f8dc67eb8c4d Author: Suyuchen Date: Mon Jun 10 18:49:47 2024 -0400 update vcr_wiki tasks in README.md commit 205721e0aad76dde30255e56149bbed121883356 Author: Suyuchen Date: Mon Jun 10 18:43:15 2024 -0400 update vcr_wiki tasks commit db8e718b502469e8536ee359c5559de87635ffc7 Author: tianyu-z Date: Mon Jun 10 16:13:58 2024 -0400 include the try-except logic for spacy commit 427dabb790118f538b64e4e5bf6a7aab9689b3d9 Author: Suyuchen Date: Mon Jun 10 15:51:05 2024 -0400 add crossed_text to vcr_wiki output commit 043b483eb55f7be4fea75c9bc0b9b03d251b109b Author: tianyu-z Date: Mon Jun 10 15:47:00 2024 -0400 switch logic commit e1f04db8f58dd10591fde335ea13f74cda7c79bd Author: tianyu-z Date: Mon Jun 10 02:38:21 2024 -0400 modify the form of VCR commit 96e8d9867c9549ab7490f4b12cfeb6a06238e0aa Author: tianyu-z Date: Mon Jun 10 00:10:30 2024 -0400 init include vcr commit 374590be62f988a76cf6704cfe394cd8ae7d4cb6 Merge: 504685e2 cb3b9ce7 Author: Kaichen Zhang - NTU Date: Fri Jun 7 20:25:48 2024 +0800 Merge pull request #101 from Gumpest/main Update conbench in README commit 504685e20b17659b913cf46f3012c16bf429e09d Author: Li Bo Date: Thu Jun 6 15:42:15 2024 +0800 Update README.md commit cb3b9ce71411da862ff01342a9122a3c656ffbd1 Merge: c9793b38 67b64ea4 Author: Yuan Zhang <56063339+Gumpest@users.noreply.github.com> Date: Thu Jun 6 11:22:24 2024 +0800 Merge branch 'EvolvingLMMs-Lab:main' into main commit c9793b3883714f254a700230b7bee781d6110e73 Author: Yuan Zhang Date: Thu Jun 6 11:21:05 2024 +0800 update README commit 67b64ea44a5a39d96c7a196a8a8345a7486bd912 Merge: 8ee7848a 5fd68451 Author: Li Bo Date: Wed Jun 5 23:12:58 2024 +0800 Merge pull request #100 from Gumpest/main add Conbench commit 5fd684515c55ef643726c1b6c720c7cbd2183ba1 Author: Yuan Zhang Date: Wed Jun 5 21:52:31 2024 +0800 add conbench commit 8ee7848aaa6383aa1f919c3f21199c81db3fff89 Merge: 747e1978 6fefaf7c Author: Li Bo Date: Tue Jun 4 17:09:33 2024 +0800 Merge pull request #95 from AtsuMiyai/new_task/upd add MM-UPD commit 747e19782996065cdce7157ee8c5e15beb5b6c59 Merge: 4854a34d 05843072 Author: Li Bo Date: Tue Jun 4 17:09:04 2024 +0800 Merge pull request #97 from CaraJ7/update Add MathVerse in README.md commit 6fefaf7cea504e35583ee7217449da290295a7a4 Author: AtsuMiyai Date: Tue Jun 4 17:36:39 2024 +0900 update utils.py for leaderboard submission commit 5f4fe360def1c48ea0cb1da6409d192784882308 Author: AtsuMiyai Date: Sun Jun 2 23:28:27 2024 +0900 slightly change query_prompt for the reproduction commit 05843072d608b970bcada1cd0db65a3c80864060 Author: CaraJ7 <1350074492@qq.com> Date: Sun Jun 2 17:05:28 2024 +0800 Add MathVerse in README.md commit 0581ab3cfb362e2024988b46fbbb00324f1233c9 Author: AtsuMiyai Date: Fri May 31 16:09:45 2024 +0900 merge model_specific_prompt_kwargs and dataset_name into each task yaml commit 4854a34d4d37efb5e201f2691ecdb054590cf20b Author: Pu Fanyi Date: Sat May 4 19:23:39 2024 +0800 Group MMMU images into one image (#83) * update * update font * Add matplotlib.font_manager import in utils.py * Refactor font handling in add_order_label function in utils.py * group mmmu --------- Co-authored-by: Li Bo commit d224794c49520f4d28a31862cf977198cd6cbc5e Author: AtsuMiyai Date: Wed May 29 15:15:59 2024 +0900 add upd commit 453e7936424220f02b99517059ca71babfbe5f5a Author: AtsuMiyai Date: Wed May 29 15:03:30 2024 +0900 add upd commit 909edd6769ddcf8a546be4fdd129416687516878 Author: AtsuMiyai Date: Wed May 29 12:52:21 2024 +0900 add upd commit 7c1ac9706cafc4801fa4da181d2f610b7838c7b8 Author: AtsuMiyai Date: Wed May 29 12:50:32 2024 +0900 add upd commit 811301c5280ddd74986645086f026ab730c8848c Author: AtsuMiyai Date: Wed May 29 12:46:58 2024 +0900 add upd commit 71401bafd1d515f704f86ab4817a758542bc4672 Author: AtsuMiyai Date: Wed May 29 12:41:21 2024 +0900 add upd commit 24dc435908d921e9f1a5706e3141b12e5d838d18 Author: Bo Li Date: Mon May 27 10:17:32 2024 +0000 fix compatibility issue of older version llava commit 616edf43731415b35f0f5e97748ed2e017a2891d Author: Bo Li Date: Mon May 27 09:32:26 2024 +0000 [Fix] import issues of multilingual llava and olympiadbench commit 4c5a99e21a63fb0ee1c7d15546d18066e1d9894b Merge: 45c05b2b b05c3e22 Author: Li Bo Date: Mon May 27 14:19:53 2024 +0800 Merge pull request #87 from vfragoso/vifragos/phi3v Adding microsoft/Phi-3-vision-128k-instruct model. commit b05c3e222fabd308dd7af4e04c1c6a0812962fe6 Author: Victor Fragoso Date: Fri May 24 16:36:37 2024 +0000 Adding documentation of Phi3v class. commit c2008971308ce8168d57c24d00b725832f099244 Author: Victor Fragoso Date: Fri May 24 16:25:02 2024 +0000 Adding prompt arguments for Phi3v on MathVista-TestMini commit 7f9fb6bcc6cd24a7b8011b8753d0ea98cc2451fd Author: Victor Fragoso Date: Fri May 24 13:24:16 2024 +0000 Adding Phi3v model. commit 45c05b2b2bece76e06849a52a0d034f9c0ac2367 Author: kcz358 Date: Thu May 23 03:47:36 2024 +0000 Set printing info for llava_hf to debug level commit 53f013ed8278776551ca992562253387cc9968d2 Author: kcz358 Date: Thu May 23 03:41:39 2024 +0000 Fix pope random name in pope full commit 22520a95f13334b75eee0cf0387151067a6bf516 Author: kcz358 Date: Thu May 23 03:41:14 2024 +0000 Add separated pope tasks by category commit d1eefb1565014b47287ffa6b350229062f8f602f Author: kcz358 Date: Thu May 9 08:36:02 2024 +0000 Update gitignore commit b2b4dbd2dc13432c79208db35abf7f55c97f1790 Author: kcz358 Date: Mon May 20 07:45:11 2024 +0000 Comment out Spice in caption task so that don't need to download stanford nlp model commit 662f05ce4c62a46a83f819d3a5925a9bd20059b5 Author: kcz358 Date: Mon May 20 03:13:13 2024 +0000 Comment out parse result in xcomposer commit 09329322916bfbb604d72ddaf50441a0947f8805 Author: kcz358 Date: Thu May 16 03:55:39 2024 +0000 Fix instructblip qformer size mismatch and multi-images problem commit 557a6a3b15e07e506bc05e2cc76ff6a2f8c93964 Author: kcz358 Date: Thu May 16 03:11:41 2024 +0000 Remove redundant code in fuyu commit 6aeb5504e74ed1980b53700d8e4d4dcf7d1b38fc Author: kcz358 Date: Thu May 16 01:45:24 2024 +0000 Fix idefics2 llava in the wild bugs commit aea80e6a71f716951353e1e5d68380243396b4d6 Author: kcz358 Date: Wed May 15 11:07:35 2024 +0000 Better task list_with_num commit 3c12a080d66b9c38f615b961befca7c30f82fa39 Author: Li Bo Date: Sat May 18 02:35:52 2024 +0800 Update LICENSE commit 82317a635a4978b32e095a06cc295d0ae23661c2 Author: Li Bo Date: Sat May 18 02:29:09 2024 +0800 Update LICENSE commit a8bba1cdb51061a0d27bf9a98cca1505b5c58ea5 Author: Li Bo Date: Sat May 18 02:28:03 2024 +0800 Create LICENSE commit caa5893b5fd2c1d32c72b97f371ccd9a8d9ec3a0 Merge: c0944486 423b0060 Author: Li Bo Date: Mon May 13 11:45:26 2024 +0800 Merge pull request #73 from EvolvingLMMs-Lab/kc/qwen_vl_api [Feat] Add qwen vl api commit c09444860362a136f17641f8b2a1f91c2bbc3715 Author: kcz358 Date: Sat May 11 06:11:19 2024 +0000 Fix llava_hf image tokens number issue commit 64f07e497f53e5bcbe9e8fb5830cc7a1daaf7ff1 Author: kcz358 Date: Thu May 9 02:04:10 2024 +0000 Fix endless warning for llava_hf generation commit 8aaa828108da8514dd9cd23a9d6d83a8b67f2d65 Author: Bo Li Date: Thu May 2 06:13:56 2024 +0000 Add model_name parameter to Llava constructor commit 7847dc4d8efe60605102414bb071b1da9851228e Author: kcz358 Date: Tue May 7 03:15:59 2024 +0000 Parse result for llava_hf 1.6 commit 3e56b4f92db39a2ce92903b0c43a34f1d14d59ec Author: kcz358 Date: Tue May 7 03:09:56 2024 +0000 Fix llava_hf generation for 1.6 commit fa3ff92b07ea5aaa633a2039818c310744f84d07 Author: kcz358 Date: Mon May 6 08:32:57 2024 +0000 Fix llava conv template for llama3 commit 423b00606aa77fd6b324c19e3d480b73ab852db6 Author: kcz358 Date: Sun May 5 07:54:52 2024 +0000 Add qwen vl api commit b7fd7a9f7aa3c0e1e50374047dfffc46a7462b90 Merge: 986139a9 c5a130b6 Author: Li Bo Date: Sun May 5 13:19:48 2024 +0800 Merge pull request #59 from EvolvingLMMs-Lab/add_idefics2 add idefics2 commit 986139a9a31154679bdea029b09639f84712db27 Merge: b46239ca 8d3526c0 Author: Li Bo Date: Fri May 3 01:18:18 2024 +0800 Merge pull request #36 from cocoshe/main [Fix] repr llava doc commit b46239cabab7b545ec99d9eae6c851e531b18374 Merge: bc69a744 373265f2 Author: Li Bo Date: Fri May 3 01:17:34 2024 +0800 Merge pull request #56 from gagan3012/main Multilingual LLava bench commit bc69a744d2cffeb06eba62e843bcc7869e27613a Merge: eef3aeb6 626e8a91 Author: Li Bo Date: Fri May 3 01:12:14 2024 +0800 Merge pull request #70 from hunterheiden/hsh/new_task/WebSRC Bugfix: WebSRC should be token-level F1 NOT character-level commit 626e8a91a4af2dd5dd774fc130cc2f4d74b2bc37 Author: Hunter Heidenreich Date: Thu May 2 09:31:03 2024 -0400 Bugfix: WebSRC should be token-level F1 NOT character-level commit eef3aeb6ab589bb1d5045af5b5c1984a69402d19 Merge: c4e9dd9f 9bca4413 Author: Li Bo Date: Thu May 2 14:38:17 2024 +0800 Merge pull request #69 from hunterheiden/hsh/new_task/WebSRC [New Task] WebSRC (multimodal Q&A on web screenshots) commit 9bca441376325173128e5c50087f068e519c48da Author: Hunter Heidenreich Date: Wed May 1 11:07:29 2024 -0400 Add code to enable compilation of submission for WebSRC test split commit 7687495b1ed552eeba088cb9ad5aaf1170e7fff9 Author: Hunter Heidenreich Date: Wed May 1 10:47:32 2024 -0400 Draft and validate websrc eval on dev split commit 4eebd3e5d7ab3b8c3116eea57318db72d2ce32bb Author: Hunter Heidenreich Date: Wed May 1 10:46:54 2024 -0400 Update main README with new task names commit 35fe80b67656114a8824eb59574089663bdc4c9a Author: Hunter Heidenreich Date: Wed May 1 10:46:20 2024 -0400 Draft README for WebSRC commit 955bd0635cc6c14a96ad869f1002e6dbefdc5071 Author: Hunter Heidenreich Date: Tue Apr 30 10:16:21 2024 -0400 Init webSRC commit c4e9dd9f6e40e8586587c4a75987aa109a37f14b Merge: d8a3a99f 319afccb Author: Li Bo Date: Fri Apr 26 14:37:22 2024 +0800 Merge pull request #63 from hunterheiden/hsh/new_task/screenspot New Task: ScreenSpot - Grounding (REC) and instruction generation (REG) on screens commit 319afccbe713ddf40a8a6fa28501e64c0ad34725 Author: Hunter Heidenreich Date: Thu Apr 25 11:44:34 2024 -0400 slight update commit 2f3811ca1bbad6a441016b05fde09a571900fca8 Author: Hunter Heidenreich Date: Thu Apr 25 11:41:04 2024 -0400 Add README file specific to ScreenSpot commit 28962cbe83631ec5d6481aaea4907a7c96fec848 Author: Hunter Heidenreich Date: Wed Apr 24 11:52:33 2024 -0400 Update README to reflect new tasks commit e457cfb4f2d6869e8367d6d5b03ad25ee4acc363 Author: Hunter Heidenreich Date: Tue Apr 23 18:33:16 2024 -0400 Create ScreenSpot on clean branch commit d8a3a99ff6142fe101fa3c188cc7f29593c44345 Merge: 3dcd0158 ed171293 Author: Li Bo Date: Tue Apr 23 10:34:03 2024 +0800 Merge pull request #61 from tupini07/patch-1 Fix typo in Qwen-VL that was causing "reference before assignment" commit ed171293d1e82075c5c6a847fc91ecbfd45cf89f Author: Andrea Tupini Date: Mon Apr 22 14:56:41 2024 -0600 refactor query construction for clarity commit cd874201c46f32a2903ddffae85f9db73e14adfd Author: Andrea Tupini Date: Mon Apr 22 14:54:29 2024 -0600 convert contexts to list if necessary and remove unnecessary construction of `questions` commit 85573674e90c8d505312ba18c5102e0051255078 Author: Andrea Tupini Date: Mon Apr 22 14:47:33 2024 -0600 Fix typo in qwen_vl that was causing "reference before assignment" commit 3dcd01582b719555bcf8eb25d91cc5e42abd2c5f Merge: 95df9fee 743673a1 Author: Li Bo Date: Sat Apr 20 22:03:16 2024 +0800 Merge pull request #60 from CaraJ7/main Add MathVerse commit 743673a1419b6e729e18c96f148745cc739d4c71 Merge: c1a54721 95df9fee Author: CaraJ7 <1350074492@qq.com> Date: Sat Apr 20 21:49:02 2024 +0800 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval commit c1a5472135c3b84061b64d997ab50dda0412ba4f Author: CaraJ7 <1350074492@qq.com> Date: Sat Apr 20 21:45:34 2024 +0800 Add MathVerse commit 373265f24e7a89cbd49ab724a2e388cc0930be78 Author: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Date: Fri Apr 12 17:21:39 2024 -0700 Add files via upload commit d8530514a5ef9378d2adeaceb228b60ec25a6718 Author: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Date: Fri Apr 12 17:19:49 2024 -0700 Create README.md commit 22a4958e993463edff352ac033014f9a485706cc Author: Bo Li Date: Thu Apr 4 17:12:43 2024 +0000 [WIP] adding mmbench dev evaluation (#75) * WIP * Update GPT evaluation model name and sys prompt * 🛠️ Scale accuracy to percentage The accuracy value is now multiplied by 100 in the aggregation function to represent it as a percentage. Regarding the evaluation process, `math` module importation and refactoring reduce progress log verbosity by logging every 100 evaluations instead of 10. It prevents potential logging overflow. Handling of NaN values is added to ensure 'default_value' is set in case of missing data, avoiding errors in split, category, and l2-category assignments. Finally, reporting of categorical and l2-categorical accuracies is streamlined through a new `calculate_hit_rates` function, improving code readability and maintenance. Issue refs: #1427, #1533 * Update GPT evaluation model name and API configuration * Refactor MMBench_Evaluator class to handle missing columns * Add print statements for detailed results in MMBench-CN(CC), MMBench-CN(Dev), and MMBench-EN(Dev) evaluations * Refactor MMBench-CN and MMBench-EN evaluation functions * 🔄 Refactor result processing and logging logic - Simplified the result processing functions across different utility modules (`cc_utils.py`, `cn_utils.py`, `en_utils.py`) to unify the handling of multiple-choice options. Now, all options ("A" to "E") are dynamically added to the result data, and default to "nan" if not provided in the document. - Removed redundant keys directly from the process results dict creation to avoid clutter and align with the new dynamic addition of options. - In `mmbench_evals.py`, removed the unnecessary check for all splits being 'dev' and streamlined the evaluation loop by eliminating the progress bar (tqdm) for a cleaner log output. - Commented-out code and verbose logging during evaluation, which may have interfered with performance, has been removed for a more efficient and less intrusive logging experience. This cleanup reduces redundancy in the codebase and improves evaluation performance. Refs #2045 --------- Co-authored-by: Bo Li (cherry picked from commit a19278c2ea6ddcbca64d3cc7f4efec7fe5775121) commit 8d3526c0869f0ad7747ff6bb02441140792b461c Author: cocoshe <1228759711@qq.com> Date: Thu Mar 28 13:38:36 2024 +0800 fix doc * feat: Add LlavaOneVision model to available models chore: Update sqlitedict dependency to version 2.1.0 * Revert "Squashed commit of the following:" This reverts commit 11b00999df3c43cb225482e030b791b2d454124c. * Refactor available models in lmms_eval Remove duplicate entries for "llava_hf", "llava_onevision", and "longva" in the AVAILABLE_MODELS dictionary in lmms_eval/models/__init__.py. * fix: Handle import errors in lmms_eval models/__init__.py The code changes in this commit fix the handling of import errors in the lmms_eval/models/__init__.py file. Previously, when an import error occurred, the code simply ignored it. This commit updates the code to log an error message using the logger module when an import error occurs. This commit also removes duplicate entries for "llava_hf", "llava_onevision", and "longva" in the AVAILABLE_MODELS dictionary. Recent user commits: - Refactor available models in lmms_eval - Revert "Squashed commit of the following:" - feat: Add LlavaOneVision model to available models - chore: Update sqlitedict dependency to version 2.1.0 * fix: Handle import errors in lmms_eval models/__init__.py * chore: Remove unused imports in lmms_eval/models/__init__.py and lmms_eval/tasks/vcr_wiki/utils.py * Remove unused imports in lmms_eval/tasks/vcr_wiki/utils.py * chore: Update lmms_eval/tasks/vcr_wiki/utils.py This commit updates the `lmms_eval/tasks/vcr_wiki/utils.py` file. It removes unused imports and fixes the condition for loading Spacy models based on the `load_package` value in the config file. Additionally, it adds a debug log message when the Spacy models are not loaded due to `load_package` being set to False. Remove unused imports in `lmms_eval/tasks/vcr_wiki/utils.py` * feat: Add new subtasks to overall score calculation The code changes in this commit add new subtasks to the overall score calculation in the `overall_score` function. The subtasks "ScanQA", "BLINK", "MathVerse", "SciVerse", and "Mantis" are included in the `categories` dictionary. This ensures that the scores for these subtasks are calculated and included in the evaluation results. Remove unused imports and update subtask categories in `utils.py` * feat: Add new subtasks to overall score calculation * chore: Update lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml Update the image aspect ratio in the default template for the llava_interleave_bench task. Change the value of "image_aspect_ratio" from "original" to "pad". This ensures that the generated images have a padded aspect ratio. * if no response directly return 0 * Squashed commit of the following: commit b2a009b6bbf8353172f5a1dd9c29ea1f67610c02 Author: Pu Fanyi Date: Mon Jul 15 19:12:25 2024 -0700 if no response directly return 0 (#142) commit 5fc5f2f5acf454fc99448b0d62eb52b4bffba0d5 Author: Kaichen Zhang - NTU Date: Tue Jul 16 10:12:11 2024 +0800 Add Muirbench (#143) * handle gen kwargs in internvl2 * Add muirbench * Add files via upload (cherry picked from commit 557083a156c3dd67ac79e22b4202e9b69b6b00f4) * update --------- Co-authored-by: Fanyi Pu Co-authored-by: Yan Shu <570533048@qq.com> commit b2a009b6bbf8353172f5a1dd9c29ea1f67610c02 Author: Pu Fanyi Date: Mon Jul 15 19:12:25 2024 -0700 if no response directly return 0 (#142) commit 5fc5f2f5acf454fc99448b0d62eb52b4bffba0d5 Author: Kaichen Zhang - NTU Date: Tue Jul 16 10:12:11 2024 +0800 Add Muirbench (#143) * handle gen kwargs in internvl2 * Add muirbench commit 4f8db1d37b1f824432927e74d6d82e06bb5aaed1 Author: Pu Fanyi Date: Fri Jul 12 17:26:50 2024 -0700 Upload live_bench results (#140) * upload results * add a readme * chore: Update upload_results.py script to use shell syntax * Update upload_results.py * Update upload_results.py commit 18f3812c4f9af2e49af6b50e8afe7f607b8a75d6 Author: Pu Fanyi Date: Wed Jul 10 18:13:43 2024 -0700 Load tasks only one time (#139) * chore: Initialize tasks only once to avoid re-initialization * chore: Initialize tasks only once to avoid re-initialization * chore: Refactor task initialization to avoid re-initialization * chore: Update task initialization to fix include_path issue * chore: Update task initialization to fix include_path issue * chore: Remove unnecessary line in muirbench.yaml * chore: Remove unnecessary line in muirbench.yaml and update gitignore * chore: Update lmms_eval to use correct variable name for world size * Update mmvet * chore: Update lmms_eval to use correct variable name for world size * chore: Remove unused lmms_eval configuration file * refactor: Update lmms_eval to handle both image and video tasks This commit updates the `Llava_OneVision` class in `llava_onevision.py` to handle both image and video tasks. It introduces conditional logic to differentiate between the two types of tasks and process the input accordingly. Additionally, it sets the image aspect ratio based on the number of visual inputs and the configuration settings. Closes #123 * Fix llava onevision loglikelihood video bug (cherry picked from commit f96e3e69fe86dcd9cb33d2bc18cc4ff2003de8be) * refactor: Update mm_spatial_pool_mode to use bilinear interpolation This commit updates the `mm_spatial_pool_mode` parameter in the `Llava_OneVision` class of `llava_onevision.py` to use bilinear interpolation instead of the previous average pooling mode. This change improves the spatial pooling process for the model. Closes #456 * chore: Update pyproject.toml with protobuf dependency version 3.20 * Squashed commit of the following: commit e106f49ceeb295fd4c89a0877073bc01b4b77c5f Author: Fanyi Pu Date: Thu Jul 25 08:14:03 2024 +0800 livebench_july commit a16295653fdda20d5e8c41c549d731ec422013e3 Author: Fanyi Pu Date: Mon Jul 22 15:09:58 2024 +0800 websites commit 2cdc06ffe6ba53a4c707c1acf9fc5f2e7886b2b8 Author: Fanyi Pu Date: Sun Jul 21 15:34:39 2024 +0800 everything use gpt-4o commit e67538d65526c58903d9e62d1914ebd39924ab67 Author: Fanyi Pu Date: Sun Jul 21 14:29:55 2024 +0800 chore: Update dataset capture settings in create_dataset.py commit 0a3bb33d37cda05bb7bfba4ecf873c2860092a03 Author: Fanyi Pu Date: Sun Jul 21 01:58:14 2024 +0800 gpt-4-turbo => gpt-4o commit 837f8b0400f04f4367f8f8f954afd64666d62fc6 Author: Fanyi Pu Date: Sat Jul 20 16:48:04 2024 +0800 chore: Update dataset name and version for live_bench task commit fa58e730978b5536005c8bd0291abbeddd761205 Author: Fanyi Pu Date: Sat Jul 20 15:05:13 2024 +0800 generate data commit faa96227a7af7bd6546578b2db68dce2acbc2c0c Author: Fanyi Pu Date: Sat Jul 20 13:15:18 2024 +0800 fix commit 60ea7ddb4fcd9f08013cd0d5b9dd8090f7e6b83e Author: Fanyi Pu Date: Sat Jul 20 13:12:31 2024 +0800 fix bugs commit 827d69d0bf967f5d69bfbee9848b4d568ca853b1 Author: Fanyi Pu Date: Sat Jul 20 08:39:41 2024 +0800 use claude to generate commit b7e2619d1a51144cd434861ac151187aed82c8c4 Author: Fanyi Pu Date: Sat Jul 20 07:36:59 2024 +0800 extract information commit f87d55d47cb0d6653765e9e3f988f4bc186f7d4c Author: Fanyi Pu Date: Sat Jul 20 07:24:07 2024 +0800 claude auto detect json mode commit dfdba507b5fbe985b0030ffec575f9f2638bc1ed Author: Li Bo Date: Tue Jul 16 11:13:52 2024 +0800 merge ov evals (#144) * chore: Update gpt_eval_model_name to "gpt-3.5-turbo" in mathvista.yaml * Squashed commit of the following: commit 994c9f97a2f8db3e9b7d7933d1e1680acde5b70b Author: Yan Shu <570533048@qq.com> Date: Mon Jul 8 17:21:23 2024 +0800 Add files via upload * Squashed commit of the following: commit e31cd7883d4555c7530795c7f102b8d78cbd372f Author: Bo Li Date: Wed Jul 10 12:08:08 2024 +1000 chore: Update lmms_eval/models/vila.py and lmms_eval/tasks/__init__.py commit 1d8c980d1089f9d7702c3b92d5c85039f2809c6d Author: kcz358 Date: Tue Jul 9 02:08:52 2024 +0000 Rename xcomposer 4KHD commit 6da76f36ecf5f9aa73057e767a4fcb60c99ff896 Author: Bo Li Date: Tue Jul 9 11:55:56 2024 +1000 Upgrade lmms-eval to version 0.2.1 commit cd1858523fcd8630082cbefba8710e0de3ee8805 Author: Bo Li Date: Tue Jul 9 11:52:23 2024 +1000 Upgrade lmms-eval to support more models and evaluation tasks commit 672d7e5bb49dcb34e1b2fdeb09f3f4588dc583a6 Author: Bo Li Date: Tue Jul 9 11:43:41 2024 +1000 feat: Add tie_weights parameter to Llava model initialization commit 2037a86261b55fa42b8ba3a04eab192b3e69d6ea Merge: e6844db1 a5c18692 Author: Bo Li Date: Tue Jul 9 11:37:12 2024 +1000 Fix gen kwargs image aspect ratio in internvl2 commit a5c186925de989b616f58a35ece36065a32b4594 Merge: 2ebec77f 557083a1 Author: Li Bo Date: Tue Jul 9 09:15:56 2024 +0800 Merge pull request #137 from shuyansy/main add MLVU task commit 557083a156c3dd67ac79e22b4202e9b69b6b00f4 Author: Yan Shu <570533048@qq.com> Date: Mon Jul 8 16:56:50 2024 +0800 Add files via upload commit 2ebec77f5606d79e9a7b995970e32792050606a1 Merge: 211bfede b23d349e Author: Li Bo Date: Mon Jul 8 11:53:06 2024 +0800 Merge pull request #136 from Dousia/main Add detailcaps commit b23d349e46d60dc149ffaa54d6e019f4996ed92d Author: ByteDance Date: Sun Jul 7 23:24:19 2024 +0800 Add install capture_metric in env commit c6e211d5f9dbb7572d3a141b6504cb1ca2007c33 Author: ByteDance Date: Sun Jul 7 23:04:13 2024 +0800 Add detailcaps commit 211bfedebad243ef82a8b0be36c3b5a9b9cb2f72 Merge: 7c208b76 79514eee Author: Li Bo Date: Tue Jul 2 23:05:12 2024 +0800 Merge pull request #133 from EvolvingLMMs-Lab/dev/wild_vision Add wild vision bench commit 79514eeebcfd6f655be2a10c776037d12a7b7214 Author: kcz358 Date: Mon Jul 1 15:10:02 2024 +0000 Fixing handling None filtered score commit 725fac2781446958b905e1e6c6eb3c0a8e582e49 Author: kcz358 Date: Mon Jul 1 08:25:42 2024 +0000 Fixing dataset name commit 8d963e132ac03fc0d835d480cfcfcabe72af143c Author: kcz358 Date: Mon Jul 1 08:24:51 2024 +0000 Fixing scoring logic commit e2990d0a69e876721256fdf946c68ba7ae0cbdc1 Author: kcz358 Date: Mon Jul 1 06:06:57 2024 +0000 Hardcode to keep image for wild vision commit ed381736730d8fb785b4ee919fdb751734ecef25 Author: kcz358 Date: Mon Jul 1 06:06:38 2024 +0000 Add wild vision 0617 commit 7c208b76640c986cfe94233dce735c3ca4ad4319 Author: Li Bo Date: Mon Jul 1 11:53:31 2024 +0800 Update README.md commit 39d40dea47bc59ff04e8b0cbc445345098debc9a Merge: e19b43a3 ba7081c0 Author: Li Bo Date: Mon Jul 1 11:47:09 2024 +0800 Merge pull request #129 from Dannoopsy/mmbench_ru add task MMBench-ru commit e19b43a3a1e7212e623061b164b0419cc0dda689 Merge: 11fd7e3f a0de8970 Author: Li Bo Date: Mon Jul 1 11:46:58 2024 +0800 Merge pull request #128 from Dannoopsy/gqa-ru add task gqa-ru commit 11fd7e3fc05908aeb01e4a6161a7b55cd38b3122 Merge: 383e7fea a7522592 Author: Li Bo Date: Mon Jul 1 11:46:16 2024 +0800 Merge pull request #130 from lscpku/vitatecs Add task VITATECS commit a75225926e5954f85466d257f99acf0163fde596 Author: lscpku Date: Fri Jun 28 20:37:06 2024 +0800 create new task vitatecs commit ba7081c0abac840002d320e30733e891298dfa11 Author: Dannoopsy <63581325+Dannoopsy@users.noreply.github.com> Date: Fri Jun 28 12:21:05 2024 +0300 change prompt to ru commit 27ea9c0055a8abf3a8198829b8617018479918e2 Author: Dannoopsy Date: Thu Jun 27 17:17:29 2024 +0000 add mmbench_ru_dev commit 383e7fead3138aedf62e9c0ec48303835ef26e2a Merge: 06fa000f ed2e7f79 Author: Li Bo Date: Fri Jun 28 00:14:10 2024 +0800 Merge pull request #126 from lorenzomammana/feature/external-package-integration External package integration using plugins commit ed2e7f792151d21bce8f1c498270b9391e1d5c85 Merge: 03947e14 06fa000f Author: Lorenzo Mammana Date: Thu Jun 27 15:38:10 2024 +0000 Merge branch 'main' into feature/external-package-integration commit a0de89708d5e6f259bb17f0eaace3c5b901b275c Author: Dannoopsy Date: Tue Jun 25 11:11:37 2024 +0000 new task gqa-ru commit 06fa000f60d3e4d160fac8ceb9959ae92a98f752 Author: kcz358 Date: Tue Jun 25 06:41:13 2024 +0000 Fix vid mme post prompt issue commit b388d79e0df6f60068196cb7047453ebd22d6ef1 Author: Li Bo Date: Sun Jun 23 22:31:16 2024 +0800 Update activitynetqa_generation.yaml commit 8f9d620fcd9d0a0742ee6bcf51ea63bd6b088a36 Author: Li Bo Date: Sun Jun 23 14:02:25 2024 +0800 Update pyproject.toml commit 6341b7c15ce9fb28eb06b067ddb299d6cf2e16c3 Merge: fce85f1b 903b042b Author: Li Bo Date: Sun Jun 23 14:02:02 2024 +0800 Merge pull request #125 from EvolvingLMMs-Lab/dev/interleave [Model] aligned llava-interleave model results on video tasks commit 903b042be016016d4ebeecb07701f3076a2d323c Author: kcz358 Date: Sat Jun 22 12:07:13 2024 +0000 Remove unnecessary lines for video llava commit d78ec86407b729a964906a8c2e50704b4bc74d06 Merge: ebe7217a fce85f1b Author: Li Bo Date: Sat Jun 22 13:57:31 2024 +0800 Merge branch 'main' into dev/interleave commit ebe7217a486c1e754e42c2cbdb834e09fbbcc9b0 Author: kcz358 Date: Sat Jun 22 02:57:08 2024 +0000 Delete unnecessary lines commit 120c474b056f9177c74e1fd9691d59e2f234b785 Author: kcz358 Date: Fri Jun 21 08:38:41 2024 +0000 Revise model registry for llava_hf and longva commit 7d6201f921088afd3f52a35076e3c6fcc9aa518c Author: kcz358 Date: Fri Jun 21 08:38:24 2024 +0000 Add longva commit 12f480699c71a12a24d4349d9b0681933201a3a6 Author: kcz358 Date: Fri Jun 21 08:35:39 2024 +0000 Remove unnecessary lines since use batched visuals now in llava commit 12cea76f1f0f14b1fd1007c9d39a9b0557368637 Author: Bo Li Date: Thu Jun 20 18:15:32 2024 +0000 chore: Add loguru for logging in lmms_eval package commit 03947e14a46fd25b412931f7c9c25f4a2971d0b4 Author: Lorenzo Mammana Date: Wed Jun 5 13:40:41 2024 +0000 feat: Allow including external tasks from plugins commit b80a91f73e15ddd0b0ce1322d7d121fa14030eed Author: Lorenzo Mammana Date: Wed Jun 5 13:04:55 2024 +0000 feat: Allow loading model configurations from other packages commit 8ef24740dd48a11c97eb627f2fff4aca107fef0d Author: Bo Li Date: Thu Jun 20 12:11:03 2024 +0000 chore: Remove unused models from lmms_eval package commit af38885fc2e066f5ea44388f33e07176f836fe28 Author: Bo Li Date: Thu Jun 20 12:07:09 2024 +0000 chore: Handle ImportError when importing models Handle the ImportError exception when importing models in the lmms_eval package. This change adds a try-except block to catch the ImportError and print an error message indicating the failed import. This will help with troubleshooting and identifying any issues with the model imports. commit fce85f1b03ff7043b29dee787c5d17a08dd2687a Merge: dbe63293 d94f83cb Author: Li Bo Date: Thu Jun 20 20:02:12 2024 +0800 Merge pull request #120 from EvolvingLMMs-Lab/pufanyi/hf_dataset_docs Add docs for datasets upload to HF commit dbe63293245a5141fdfd80bda7657c304f6bd32f Author: choiszt Date: Thu Jun 20 15:14:21 2024 +0800 update ablation for videomme datasets commit d94f83cb3f08b61a2c75cc4326e58792100605b3 Author: Li Bo Date: Thu Jun 20 13:30:59 2024 +0800 Update README.md commit cab8159ff35db330536c0b6dfb4b0a3b24142209 Author: Li Bo Date: Thu Jun 20 13:30:29 2024 +0800 Update README.md commit 45876652a877a8006b828f32f5cc4660629f9190 Author: kcz358 Date: Thu Jun 20 03:55:30 2024 +0000 Add llava_hf back to registry commit 3463651b8c54d36cd94169e3d376f5ed225a195a Author: kcz358 Date: Thu Jun 20 03:54:33 2024 +0000 Remove handling non-visual loop in llava commit cb0d3f49b72790b081f981e0e6147131542f7f68 Author: Fanyi Pu Date: Thu Jun 20 02:11:18 2024 +0800 update readme commit 813877bfe5ac590cdbe92dd74d18f83a2091f748 Author: Fanyi Pu Date: Wed Jun 19 15:37:52 2024 +0800 to sh script commit a14684b8557d5894976448a5c559ed7a66a6cf16 Author: Fanyi Pu Date: Wed Jun 19 15:37:04 2024 +0800 lint commit d0f8851d42ba31f5da2a7a65e91499db45174dbc Author: Fanyi Pu Date: Wed Jun 19 15:36:48 2024 +0800 small fix commit 63748e9718f287ad433afc90e340b5e17a89c1ed Author: Fanyi Pu Date: Wed Jun 19 15:36:43 2024 +0800 small fix commit 7f1159a1fe04cfb783dc31d4fbdef3bda0ce19e4 Author: Fanyi Pu Date: Wed Jun 19 15:35:05 2024 +0800 update preparation commit 19f9bd621c76a483ff98f8c7eb78f64753da683a Author: Fanyi Pu Date: Wed Jun 19 15:23:24 2024 +0800 docs commit ce6f889ba02d819979c7922f6336cf4f1f718f65 Author: Fanyi Pu Date: Wed Jun 19 15:04:16 2024 +0800 tutorial commit f513c520c2a3dad26d2b2ca5c4ed4db05a493c73 Author: Bo Li Date: Wed Jun 19 06:51:19 2024 +0000 chore: Update dependencies to fix potential risks and improve compatibility commit efb529552c5e4ba039a4cba8e9aa5cb7ba65bf90 Author: kcz358 Date: Wed Jun 19 10:25:58 2024 +0800 Release llava-wilder commit 742651fc9daf97e2f57831ed6e6e7ee7ead7d555 Author: Fanyi Pu Date: Wed Jun 19 07:44:26 2024 +0800 feat: Add support for auto downloading tar format videos commit 511b6259828212fcba954cdeb8cf90d6e5daabf8 Merge: 22a4958e 050b2c37 Author: Bo Li Date: Tue Jun 18 17:01:03 2024 +0000 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval commit 050b2c370017e9b97475dd6cf01fd051b5ca5c86 Merge: 74facb41 ef306512 Author: Li Bo Date: Tue Jun 18 13:13:38 2024 +0800 Merge pull request #114 from zjysteven/add-tinyllava add tinyllava commit ef306512e5135f76dffa383f600b8733015836e8 Author: Jingyang Zhang Date: Mon Jun 17 17:57:02 2024 -0400 fix typo commit 9bab67732a4238097725deddf867fb1946ffee40 Merge: dbfb2387 74facb41 Author: Jingyang Zhang Date: Sun Jun 16 10:56:05 2024 -0400 Merge branch 'EvolvingLMMs-Lab:main' into add-tinyllava commit 74facb41a826691dfce4458cf1d8659b34fc5bf5 Merge: 8ba192f9 d5df72de Author: Li Bo Date: Sun Jun 16 17:59:19 2024 +0800 Merge pull request #118 from teowu/main Fix the potential risk by PR #117 commit d5df72de2d03108d6b365818ecc3551ac9aa6302 Merge: 5bf59ed2 8ba192f9 Author: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Date: Sun Jun 16 15:32:13 2024 +0800 Merge branch 'EvolvingLMMs-Lab:main' into main commit 5bf59ed250da98a408a94e214a73caa400cba842 Author: teowu Date: Sun Jun 16 07:27:28 2024 +0000 fix #117, allow auto download with tar format videos commit 98b3955cb808e36303c030aea78eb037d1ec59ce Merge: a056f118 be9dada8 Author: teowu Date: Sun Jun 16 07:25:07 2024 +0000 Merge branch 'main' of https://github.com/teowu/lmms-eval into main commit a056f118704eccec86ce32ab86981ce4bc1e1deb Author: teowu Date: Sun Jun 16 07:23:54 2024 +0000 fix #117, allow auto download with tar format videos commit 8ba192f94edf5d99598983445d5faa4f8807c49f Merge: 7cc28907 be9dada8 Author: Li Bo Date: Sat Jun 15 17:30:59 2024 +0800 Merge pull request #117 from teowu/main LongVideoBench for LMMs-Eval commit be9dada8b4189c53c08e1674ab273242cf2f80a0 Merge: 62ea8ceb 7cc28907 Author: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Date: Sat Jun 15 16:39:20 2024 +0800 Merge pull request #1 from EvolvingLMMs-Lab/main Merge pull request #113 from teowu/main commit 62ea8ceb223ef2b51ebab2bcd50d5cf339c35cfe Author: teowu Date: Sat Jun 15 08:30:11 2024 +0000 LongVideoBench support: image LMMs (idefics2, phi3) and video LMMs (LLaVA-Next-Video-34B) commit 7cc28907edbb4eb58ee1398772a48110ea35dd96 Merge: 4bc7224d ea14cd4b Author: Li Bo Date: Sat Jun 15 14:10:22 2024 +0800 Merge pull request #113 from teowu/main Q-Bench, Q-Bench2, A-Bench commit dbfb23873979f789477f4797ee2d6071e0fd921e Author: Jingyang Date: Fri Jun 14 16:20:42 2024 -0400 add tinyllava commit ea14cd4b361f4c95b3665cbdb95bc51754090eb5 Author: teowu Date: Fri Jun 14 15:01:52 2024 +0000 Add qbench, qbench2, abench; fix phi3v as its current implementation does not support multi-image commit 4bc7224dcd27fe8b288bfc3fed4d7a9da9635658 Merge: 2797987f bf14cb85 Author: Li Bo Date: Fri Jun 14 02:14:43 2024 +0800 Merge pull request #111 from XinrunDu/main add II-Bench commit bf14cb8527b2b7ac438a36567a875168bc02d294 Author: XinrunDu Date: Thu Jun 13 09:37:02 2024 +0000 fix dataset_path commit 6248113f4e11a0ac396d31fa1b032a142fea8cb4 Author: XinrunDu Date: Thu Jun 13 09:32:06 2024 +0000 add II-Bench commit 2797987f5b88b87bd172714b678a75a1d8051826 Merge: 63d82f1f 66d4bb2d Author: Li Bo Date: Thu Jun 13 11:14:47 2024 +0800 Merge pull request #109 from EvolvingLMMs-Lab/pufanyi/update_version [Small Update] Update the version of LMMs-Eval commit 66d4bb2d9c9afbbdea40196d4ad80e214d0b14b6 Author: Fanyi Pu Date: Thu Jun 13 11:13:00 2024 +0800 update version commit 63d82f1ff11eb430d91a15d6788a1f0b4d596850 Author: Li Bo Date: Thu Jun 13 11:04:32 2024 +0800 Update README.md commit 44a33799671cb668f55366d5e5a4ddb051a3a1b4 Merge: 5ed00356 0ce46d08 Author: Li Bo Date: Thu Jun 13 04:00:12 2024 +0800 Merge pull request #105 from tianyu-z/main Include VCR commit 0ce46d088e473d12d63de44f17c67dceab25658c Author: Suyuchen Date: Wed Jun 12 15:56:34 2024 -0400 update README.md commit 46a88d8b0199ed44d2ff459fb372f2e006960cea Merge: 47b13b9b 5ed00356 Author: Suyuchen Date: Wed Jun 12 15:50:26 2024 -0400 merged readme.md commit 47b13b9b320d36ac53b3622557e31239f7c22621 Author: Suyuchen Date: Wed Jun 12 15:30:52 2024 -0400 update aggregation function for vcr_wiki commit 5ed00356676cf5d0ff056cf27d1b519b8e303ff7 Author: Li Bo Date: Thu Jun 13 03:21:42 2024 +0800 Update README.md commit ed8806839db5988ced672bd162b7b046edb4863a Author: Li Bo Date: Thu Jun 13 03:13:59 2024 +0800 Update README.md commit fea3806026932a6e2bd6e538bcc413e33abdf245 Merge: d99a24ab 05dc8e85 Author: Li Bo Date: Thu Jun 13 03:11:49 2024 +0800 Merge pull request #108 from EvolvingLMMs-Lab/internal_main_dev [Upgrade to v0.2] Embracing Video Evaluations with LMMs-Eval commit 05dc8e853eab7c6bc782a1e2662d2efe7422f767 Author: Bo Li Date: Wed Jun 12 15:56:04 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit cbeee20bc4ffb510a2b23d96cdaf4077be7c2a9e Author: Bo Li Date: Wed Jun 12 15:50:30 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit f00d5498b69dd4f7e54c907ac906abc7c128f000 Author: Bo Li Date: Wed Jun 12 15:46:33 2024 +0000 Update image alignment in README.md commit 34156335db74cef9e3f0915d7172fd6b22456c15 Author: Bo Li Date: Wed Jun 12 15:43:16 2024 +0000 Update llava conv_template in lmms_eval/models/llava.py commit 50575a950736bc8fc1e191310314cbb5fdff5720 Author: Bo Li Date: Wed Jun 12 15:39:03 2024 +0000 chore: Update lmms-eval to support video evaluations for LLaVA models commit c9b2252fb8a15dd04252af5e6b4613855afd6ada Author: Bo Li Date: Wed Jun 12 15:33:48 2024 +0000 Bump version to 0.2.0.dev0 commit 465bd4205e8097e9c037b24a3ed08dd6a7694efa Merge: e43bd840 d99a24ab Author: Bo Li Date: Wed Jun 12 15:04:25 2024 +0000 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval into internal_main_dev commit e43bd840b63eb499856e36d9d2ba45c924abcead Author: Bo Li Date: Wed Jun 12 14:54:06 2024 +0000 chore: Remove unnecessary files and code related to live_bench and sft_eval tasks commit d99a24abd06df10d07e5a4d0ad5030613f92f2e7 Merge: 374590be a66003be Author: Li Bo Date: Wed Jun 12 19:45:57 2024 +0800 Merge pull request #107 from AtsuMiyai/new_task/upd_update update gpt-3.5-turbo version commit a66003befe4175824a1be6ed59f5f5b88c15f792 Author: AtsuMiyai Date: Wed Jun 12 17:05:17 2024 +0900 update gpt-3.5-turbo version commit ee91f272985f32eeb9cd6faa41afdd8eb49cac30 Author: AtsuMiyai Date: Wed Jun 12 16:50:53 2024 +0900 update gpt-3.5-turbo version commit 326b9694fc77398592b8caf3ba0bc2e2bb903813 Author: tianyu-z Date: Mon Jun 10 20:07:40 2024 -0400 include std and confidence interval commit cd050d4a721d01a2ace0cd030cf7f8dc67eb8c4d Author: Suyuchen Date: Mon Jun 10 18:49:47 2024 -0400 update vcr_wiki tasks in README.md commit 205721e0aad76dde30255e56149bbed121883356 Author: Suyuchen Date: Mon Jun 10 18:43:15 2024 -0400 update vcr_wiki tasks commit db8e718b502469e8536ee359c5559de87635ffc7 Author: tianyu-z Date: Mon Jun 10 16:13:58 2024 -0400 include the try-except logic for spacy commit 427dabb790118f538b64e4e5bf6a7aab9689b3d9 Author: Suyuchen Date: Mon Jun 10 15:51:05 2024 -0400 add crossed_text to vcr_wiki output commit 043b483eb55f7be4fea75c9bc0b9b03d251b109b Author: tianyu-z Date: Mon Jun 10 15:47:00 2024 -0400 switch logic commit e1f04db8f58dd10591fde335ea13f74cda7c79bd Author: tianyu-z Date: Mon Jun 10 02:38:21 2024 -0400 modify the form of VCR commit 96e8d9867c9549ab7490f4b12cfeb6a06238e0aa Author: tianyu-z Date: Mon Jun 10 00:10:30 2024 -0400 init include vcr commit 374590be62f988a76cf6704cfe394cd8ae7d4cb6 Merge: 504685e2 cb3b9ce7 Author: Kaichen Zhang - NTU Date: Fri Jun 7 20:25:48 2024 +0800 Merge pull request #101 from Gumpest/main Update conbench in README commit 504685e20b17659b913cf46f3012c16bf429e09d Author: Li Bo Date: Thu Jun 6 15:42:15 2024 +0800 Update README.md commit cb3b9ce71411da862ff01342a9122a3c656ffbd1 Merge: c9793b38 67b64ea4 Author: Yuan Zhang <56063339+Gumpest@users.noreply.github.com> Date: Thu Jun 6 11:22:24 2024 +0800 Merge branch 'EvolvingLMMs-Lab:main' into main commit c9793b3883714f254a700230b7bee781d6110e73 Author: Yuan Zhang Date: Thu Jun 6 11:21:05 2024 +0800 update README commit 67b64ea44a5a39d96c7a196a8a8345a7486bd912 Merge: 8ee7848a 5fd68451 Author: Li Bo Date: Wed Jun 5 23:12:58 2024 +0800 Merge pull request #100 from Gumpest/main add Conbench commit 5fd684515c55ef643726c1b6c720c7cbd2183ba1 Author: Yuan Zhang Date: Wed Jun 5 21:52:31 2024 +0800 add conbench commit 8ee7848aaa6383aa1f919c3f21199c81db3fff89 Merge: 747e1978 6fefaf7c Author: Li Bo Date: Tue Jun 4 17:09:33 2024 +0800 Merge pull request #95 from AtsuMiyai/new_task/upd add MM-UPD commit 747e19782996065cdce7157ee8c5e15beb5b6c59 Merge: 4854a34d 05843072 Author: Li Bo Date: Tue Jun 4 17:09:04 2024 +0800 Merge pull request #97 from CaraJ7/update Add MathVerse in README.md commit 6fefaf7cea504e35583ee7217449da290295a7a4 Author: AtsuMiyai Date: Tue Jun 4 17:36:39 2024 +0900 update utils.py for leaderboard submission commit 5f4fe360def1c48ea0cb1da6409d192784882308 Author: AtsuMiyai Date: Sun Jun 2 23:28:27 2024 +0900 slightly change query_prompt for the reproduction commit 05843072d608b970bcada1cd0db65a3c80864060 Author: CaraJ7 <1350074492@qq.com> Date: Sun Jun 2 17:05:28 2024 +0800 Add MathVerse in README.md commit 0581ab3cfb362e2024988b46fbbb00324f1233c9 Author: AtsuMiyai Date: Fri May 31 16:09:45 2024 +0900 merge model_specific_prompt_kwargs and dataset_name into each task yaml commit 4854a34d4d37efb5e201f2691ecdb054590cf20b Author: Pu Fanyi Date: Sat May 4 19:23:39 2024 +0800 Group MMMU images into one image (#83) * update * update font * Add matplotlib.font_manager import in utils.py * Refactor font handling in add_order_label function in utils.py * group mmmu --------- Co-authored-by: Li Bo commit d224794c49520f4d28a31862cf977198cd6cbc5e Author: AtsuMiyai Date: Wed May 29 15:15:59 2024 +0900 add upd commit 453e7936424220f02b99517059ca71babfbe5f5a Author: AtsuMiyai Date: Wed May 29 15:03:30 2024 +0900 add upd commit 909edd6769ddcf8a546be4fdd129416687516878 Author: AtsuMiyai Date: Wed May 29 12:52:21 2024 +0900 add upd commit 7c1ac9706cafc4801fa4da181d2f610b7838c7b8 Author: AtsuMiyai Date: Wed May 29 12:50:32 2024 +0900 add upd commit 811301c5280ddd74986645086f026ab730c8848c Author: AtsuMiyai Date: Wed May 29 12:46:58 2024 +0900 add upd commit 71401bafd1d515f704f86ab4817a758542bc4672 Author: AtsuMiyai Date: Wed May 29 12:41:21 2024 +0900 add upd commit 24dc435908d921e9f1a5706e3141b12e5d838d18 Author: Bo Li Date: Mon May 27 10:17:32 2024 +0000 fix compatibility issue of older version llava commit 616edf43731415b35f0f5e97748ed2e017a2891d Author: Bo Li Date: Mon May 27 09:32:26 2024 +0000 [Fix] import issues of multilingual llava and olympiadbench commit 4c5a99e21a63fb0ee1c7d15546d18066e1d9894b Merge: 45c05b2b b05c3e22 Author: Li Bo Date: Mon May 27 14:19:53 2024 +0800 Merge pull request #87 from vfragoso/vifragos/phi3v Adding microsoft/Phi-3-vision-128k-instruct model. commit b05c3e222fabd308dd7af4e04c1c6a0812962fe6 Author: Victor Fragoso Date: Fri May 24 16:36:37 2024 +0000 Adding documentation of Phi3v class. commit c2008971308ce8168d57c24d00b725832f099244 Author: Victor Fragoso Date: Fri May 24 16:25:02 2024 +0000 Adding prompt arguments for Phi3v on MathVista-TestMini commit 7f9fb6bcc6cd24a7b8011b8753d0ea98cc2451fd Author: Victor Fragoso Date: Fri May 24 13:24:16 2024 +0000 Adding Phi3v model. commit 45c05b2b2bece76e06849a52a0d034f9c0ac2367 Author: kcz358 Date: Thu May 23 03:47:36 2024 +0000 Set printing info for llava_hf to debug level commit 53f013ed8278776551ca992562253387cc9968d2 Author: kcz358 Date: Thu May 23 03:41:39 2024 +0000 Fix pope random name in pope full commit 22520a95f13334b75eee0cf0387151067a6bf516 Author: kcz358 Date: Thu May 23 03:41:14 2024 +0000 Add separated pope tasks by category commit d1eefb1565014b47287ffa6b350229062f8f602f Author: kcz358 Date: Thu May 9 08:36:02 2024 +0000 Update gitignore commit b2b4dbd2dc13432c79208db35abf7f55c97f1790 Author: kcz358 Date: Mon May 20 07:45:11 2024 +0000 Comment out Spice in caption task so that don't need to download stanford nlp model commit 662f05ce4c62a46a83f819d3a5925a9bd20059b5 Author: kcz358 Date: Mon May 20 03:13:13 2024 +0000 Comment out parse result in xcomposer commit 09329322916bfbb604d72ddaf50441a0947f8805 Author: kcz358 Date: Thu May 16 03:55:39 2024 +0000 Fix instructblip qformer size mismatch and multi-images problem commit 557a6a3b15e07e506bc05e2cc76ff6a2f8c93964 Author: kcz358 Date: Thu May 16 03:11:41 2024 +0000 Remove redundant code in fuyu commit 6aeb5504e74ed1980b53700d8e4d4dcf7d1b38fc Author: kcz358 Date: Thu May 16 01:45:24 2024 +0000 Fix idefics2 llava in the wild bugs commit aea80e6a71f716951353e1e5d68380243396b4d6 Author: kcz358 Date: Wed May 15 11:07:35 2024 +0000 Better task list_with_num commit 3c12a080d66b9c38f615b961befca7c30f82fa39 Author: Li Bo Date: Sat May 18 02:35:52 2024 +0800 Update LICENSE commit 82317a635a4978b32e095a06cc295d0ae23661c2 Author: Li Bo Date: Sat May 18 02:29:09 2024 +0800 Update LICENSE commit a8bba1cdb51061a0d27bf9a98cca1505b5c58ea5 Author: Li Bo Date: Sat May 18 02:28:03 2024 +0800 Create LICENSE commit caa5893b5fd2c1d32c72b97f371ccd9a8d9ec3a0 Merge: c0944486 423b0060 Author: Li Bo Date: Mon May 13 11:45:26 2024 +0800 Merge pull request #73 from EvolvingLMMs-Lab/kc/qwen_vl_api [Feat] Add qwen vl api commit c09444860362a136f17641f8b2a1f91c2bbc3715 Author: kcz358 Date: Sat May 11 06:11:19 2024 +0000 Fix llava_hf image tokens number issue commit 64f07e497f53e5bcbe9e8fb5830cc7a1daaf7ff1 Author: kcz358 Date: Thu May 9 02:04:10 2024 +0000 Fix endless warning for llava_hf generation commit 8aaa828108da8514dd9cd23a9d6d83a8b67f2d65 Author: Bo Li Date: Thu May 2 06:13:56 2024 +0000 Add model_name parameter to Llava constructor commit 7847dc4d8efe60605102414bb071b1da9851228e Author: kcz358 Date: Tue May 7 03:15:59 2024 +0000 Parse result for llava_hf 1.6 commit 3e56b4f92db39a2ce92903b0c43a34f1d14d59ec Author: kcz358 Date: Tue May 7 03:09:56 2024 +0000 Fix llava_hf generation for 1.6 commit fa3ff92b07ea5aaa633a2039818c310744f84d07 Author: kcz358 Date: Mon May 6 08:32:57 2024 +0000 Fix llava conv template for llama3 commit 423b00606aa77fd6b324c19e3d480b73ab852db6 Author: kcz358 Date: Sun May 5 07:54:52 2024 +0000 Add qwen vl api commit b7fd7a9f7aa3c0e1e50374047dfffc46a7462b90 Merge: 986139a9 c5a130b6 Author: Li Bo Date: Sun May 5 13:19:48 2024 +0800 Merge pull request #59 from EvolvingLMMs-Lab/add_idefics2 add idefics2 commit 986139a9a31154679bdea029b09639f84712db27 Merge: b46239ca 8d3526c0 Author: Li Bo Date: Fri May 3 01:18:18 2024 +0800 Merge pull request #36 from cocoshe/main [Fix] repr llava doc commit b46239cabab7b545ec99d9eae6c851e531b18374 Merge: bc69a744 373265f2 Author: Li Bo Date: Fri May 3 01:17:34 2024 +0800 Merge pull request #56 from gagan3012/main Multilingual LLava bench commit bc69a744d2cffeb06eba62e843bcc7869e27613a Merge: eef3aeb6 626e8a91 Author: Li Bo Date: Fri May 3 01:12:14 2024 +0800 Merge pull request #70 from hunterheiden/hsh/new_task/WebSRC Bugfix: WebSRC should be token-level F1 NOT character-level commit 626e8a91a4af2dd5dd774fc130cc2f4d74b2bc37 Author: Hunter Heidenreich Date: Thu May 2 09:31:03 2024 -0400 Bugfix: WebSRC should be token-level F1 NOT character-level commit eef3aeb6ab589bb1d5045af5b5c1984a69402d19 Merge: c4e9dd9f 9bca4413 Author: Li Bo Date: Thu May 2 14:38:17 2024 +0800 Merge pull request #69 from hunterheiden/hsh/new_task/WebSRC [New Task] WebSRC (multimodal Q&A on web screenshots) commit 9bca441376325173128e5c50087f068e519c48da Author: Hunter Heidenreich Date: Wed May 1 11:07:29 2024 -0400 Add code to enable compilation of submission for WebSRC test split commit 7687495b1ed552eeba088cb9ad5aaf1170e7fff9 Author: Hunter Heidenreich Date: Wed May 1 10:47:32 2024 -0400 Draft and validate websrc eval on dev split commit 4eebd3e5d7ab3b8c3116eea57318db72d2ce32bb Author: Hunter Heidenreich Date: Wed May 1 10:46:54 2024 -0400 Update main README with new task names commit 35fe80b67656114a8824eb59574089663bdc4c9a Author: Hunter Heidenreich Date: Wed May 1 10:46:20 2024 -0400 Draft README for WebSRC commit 955bd0635cc6c14a96ad869f1002e6dbefdc5071 Author: Hunter Heidenreich Date: Tue Apr 30 10:16:21 2024 -0400 Init webSRC commit c4e9dd9f6e40e8586587c4a75987aa109a37f14b Merge: d8a3a99f 319afccb Author: Li Bo Date: Fri Apr 26 14:37:22 2024 +0800 Merge pull request #63 from hunterheiden/hsh/new_task/screenspot New Task: ScreenSpot - Grounding (REC) and instruction generation (REG) on screens commit 319afccbe713ddf40a8a6fa28501e64c0ad34725 Author: Hunter Heidenreich Date: Thu Apr 25 11:44:34 2024 -0400 slight update commit 2f3811ca1bbad6a441016b05fde09a571900fca8 Author: Hunter Heidenreich Date: Thu Apr 25 11:41:04 2024 -0400 Add README file specific to ScreenSpot commit 28962cbe83631ec5d6481aaea4907a7c96fec848 Author: Hunter Heidenreich Date: Wed Apr 24 11:52:33 2024 -0400 Update README to reflect new tasks commit e457cfb4f2d6869e8367d6d5b03ad25ee4acc363 Author: Hunter Heidenreich Date: Tue Apr 23 18:33:16 2024 -0400 Create ScreenSpot on clean branch commit d8a3a99ff6142fe101fa3c188cc7f29593c44345 Merge: 3dcd0158 ed171293 Author: Li Bo Date: Tue Apr 23 10:34:03 2024 +0800 Merge pull request #61 from tupini07/patch-1 Fix typo in Qwen-VL that was causing "reference before assignment" commit ed171293d1e82075c5c6a847fc91ecbfd45cf89f Author: Andrea Tupini Date: Mon Apr 22 14:56:41 2024 -0600 refactor query construction for clarity commit cd874201c46f32a2903ddffae85f9db73e14adfd Author: Andrea Tupini Date: Mon Apr 22 14:54:29 2024 -0600 convert contexts to list if necessary and remove unnecessary construction of `questions` commit 85573674e90c8d505312ba18c5102e0051255078 Author: Andrea Tupini Date: Mon Apr 22 14:47:33 2024 -0600 Fix typo in qwen_vl that was causing "reference before assignment" commit 3dcd01582b719555bcf8eb25d91cc5e42abd2c5f Merge: 95df9fee 743673a1 Author: Li Bo Date: Sat Apr 20 22:03:16 2024 +0800 Merge pull request #60 from CaraJ7/main Add MathVerse commit 743673a1419b6e729e18c96f148745cc739d4c71 Merge: c1a54721 95df9fee Author: CaraJ7 <1350074492@qq.com> Date: Sat Apr 20 21:49:02 2024 +0800 Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/lmms-eval commit c1a5472135c3b84061b64d997ab50dda0412ba4f Author: CaraJ7 <1350074492@qq.com> Date: Sat Apr 20 21:45:34 2024 +0800 Add MathVerse commit 373265f24e7a89cbd49ab724a2e388cc0930be78 Author: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Date: Fri Apr 12 17:21:39 2024 -0700 Add files via upload commit d8530514a5ef9378d2adeaceb228b60ec25a6718 Author: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Date: Fri Apr 12 17:19:49 2024 -0700 Create README.md commit 22a4958e993463edff352ac033014f9a485706cc Author: Bo Li Date: Thu Apr 4 17:12:43 2024 +0000 [WIP] adding mmbench dev evaluation (#75) * WIP * Update GPT evaluation model name and sys prompt * 🛠️ Scale accuracy to percentage The accuracy value is now multiplied by 100 in the aggregation function to represent it as a percentage. Regarding the evaluation process, `math` module importation and refactoring reduce progress log verbosity by logging every 100 evaluations instead of 10. It prevents potential logging overflow. Handling of NaN values is added to ensure 'default_value' is set in case of missing data, avoiding errors in split, category, and l2-category assignments. Finally, reporting of categorical and l2-categorical accuracies is streamlined through a new `calculate_hit_rates` function, improving code readability and maintenance. Issue refs: #1427, #1533 * Update GPT evaluation model name and API configuration * Refactor MMBench_Evaluator class to handle missing columns * Add print statements for detailed results in MMBench-CN(CC), MMBench-CN(Dev), and MMBench-EN(Dev) evaluations * Refactor MMBench-CN and MMBench-EN evaluation functions * 🔄 Refactor result processing and logging logic - Simplified the result processing functions across different utility modules (`cc_utils.py`, `cn_utils.py`, `en_utils.py`) to unify the handling of multiple-choice options. Now, all options ("A" to "E") are dynamically added to the result data, and default to "nan" if not provided in the document. - Removed redundant keys directly from the process results dict creation to avoid clutter and align with the new dynamic addition of options. - In `mmbench_evals.py`, removed the unnecessary check for all splits being 'dev' and streamlined the evaluation loop by eliminating the progress bar (tqdm) for a cleaner log output. - Commented-out code and verbose logging during evaluation, which may have interfered with performance, has been removed for a more efficient and less intrusive logging experience. This cleanup reduces redundancy in the codebase and improves evaluation performance. Refs #2045 --------- Co-authored-by: Bo Li (cherry picked from commit a19278c2ea6ddcbca64d3cc7f4efec7fe5775121) commit 8d3526c0869f0ad7747ff6bb02441140792b461c Author: cocoshe <1228759711@qq.com> Date: Thu Mar 28 13:38:36 2024 +0800 fix doc * feat: Add LlavaOneVision model to available models chore: Update sqlitedict dependency to version 2.1.0 * Revert "Squashed commit of the following:" This reverts commit 11b00999df3c43cb225482e030b791b2d454124c. * Refactor available models in lmms_eval Remove duplicate entries for "llava_hf", "llava_onevision", and "longva" in the AVAILABLE_MODELS dictionary in lmms_eval/models/__init__.py. * fix: Handle import errors in lmms_eval models/__init__.py The code changes in this commit fix the handling of import errors in the lmms_eval/models/__init__.py file. Previously, when an import error occurred, the code simply ignored it. This commit updates the code to log an error message using the logger module when an import error occurs. This commit also removes duplicate entries for "llava_hf", "llava_onevision", and "longva" in the AVAILABLE_MODELS dictionary. Recent user commits: - Refactor available models in lmms_eval - Revert "Squashed commit of the following:" - feat: Add LlavaOneVision model to available models - chore: Update sqlitedict dependency to version 2.1.0 * fix: Handle import errors in lmms_eval models/__init__.py * chore: Remove unused imports in lmms_eval/models/__init__.py and lmms_eval/tasks/vcr_wiki/utils.py * Remove unused imports in lmms_eval/tasks/vcr_wiki/utils.py * chore: Update lmms_eval/tasks/vcr_wiki/utils.py This commit updates the `lmms_eval/tasks/vcr_wiki/utils.py` file. It removes unused imports and fixes the condition for loading Spacy models based on the `load_package` value in the config file. Additionally, it adds a debug log message when the Spacy models are not loaded due to `load_package` being set to False. Remove unused imports in `lmms_eval/tasks/vcr_wiki/utils.py` * feat: Add new subtasks to overall score calculation The code changes in this commit add new subtasks to the overall score calculation in the `overall_score` function. The subtasks "ScanQA", "BLINK", "MathVerse", "SciVerse", and "Mantis" are included in the `categories` dictionary. This ensures that the scores for these subtasks are calculated and included in the evaluation results. Remove unused imports and update subtask categories in `utils.py` * feat: Add new subtasks to overall score calculation * chore: Update lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml Update the image aspect ratio in the default template for the llava_interleave_bench task. Change the value of "image_aspect_ratio" from "original" to "pad". This ensures that the generated images have a padded aspect ratio. * if no response directly return 0 * Squashed commit of the following: commit b2a009b6bbf8353172f5a1dd9c29ea1f67610c02 Author: Pu Fanyi Date: Mon Jul 15 19:12:25 2024 -0700 if no response directly return 0 (#142) commit 5fc5f2f5acf454fc99448b0d62eb52b4bffba0d5 Author: Kaichen Zhang - NTU Date: Tue Jul 16 10:12:11 2024 +0800 Add Muirbench (#143) * handle gen kwargs in internvl2 * Add muirbench * Add files via upload (cherry picked from commit 557083a156c3dd67ac79e22b4202e9b69b6b00f4) * update --------- Co-authored-by: Fanyi Pu Co-authored-by: Yan Shu <570533048@qq.com> commit b2a009b6bbf8353172f5a1dd9c29ea1f67610c02 Author: Pu Fanyi Date: Mon Jul 15 19:12:25 2024 -0700 if no response directly return 0 (#142) commit 5fc5f2f5acf454fc99448b0d62eb52b4bffba0d5 Author: Kaichen Zhang - NTU Date: Tue Jul 16 10:12:11 2024 +0800 Add Muirbench (#143) * handle gen kwargs in internvl2 * Add muirbench commit 4f8db1d37b1f824432927e74d6d82e06bb5aaed1 Author: Pu Fanyi Date: Fri Jul 12 17:26:50 2024 -0700 Upload live_bench results (#140) * upload results * add a readme * chore: Update upload_results.py script to use shell syntax * Update upload_results.py * Update upload_results.py commit 18f3812c4f9af2e49af6b50e8afe7f607b8a75d6 Author: Pu Fanyi Date: Wed Jul 10 18:13:43 2024 -0700 Load tasks only one time (#139) * chore: Initialize tasks only once to avoid re-initialization * chore: Initialize tasks only once to avoid re-initialization * chore: Refactor task initialization to avoid re-initialization * chore: Update task initialization to fix include_path issue * chore: Update task initialization to fix include_path issue * Merge pull request #158 from skyil7/main Add MMStar * LiveBench July (#146) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task (cherry picked from commit 2e7fd3f7a01cfd24afd5a70c2ee21ce196823aec) * chore: Update dataset name and version for live_bench task --------- Co-authored-by: cocoshe <1228759711@qq.com> Co-authored-by: Bo Li Co-authored-by: Gagan Bhatia <49101362+gagan3012@users.noreply.github.com> Co-authored-by: CaraJ7 <1350074492@qq.com> Co-authored-by: Andrea Tupini Co-authored-by: Hunter Heidenreich Co-authored-by: kcz358 Co-authored-by: Victor Fragoso Co-authored-by: AtsuMiyai Co-authored-by: Pu Fanyi Co-authored-by: Yuan Zhang Co-authored-by: Yuan Zhang <56063339+Gumpest@users.noreply.github.com> Co-authored-by: tianyu-z Co-authored-by: Suyuchen Co-authored-by: XinrunDu Co-authored-by: teowu Co-authored-by: Jingyang Co-authored-by: Teo (Timothy) Wu Haoning <38696372+teowu@users.noreply.github.com> Co-authored-by: choiszt Co-authored-by: Lorenzo Mammana Co-authored-by: Dannoopsy Co-authored-by: Dannoopsy <63581325+Dannoopsy@users.noreply.github.com> Co-authored-by: lscpku Co-authored-by: ByteDance Co-authored-by: Yan Shu <570533048@qq.com> Co-authored-by: Hongyuan Dong <45926533+Dousia@users.noreply.github.com> --- .gitignore | 2 + README.md | 4 +- docs/current_tasks.md | 1 + lmms_eval/__main__.py | 1 - lmms_eval/api/samplers.py | 4 +- lmms_eval/evaluator.py | 18 +- lmms_eval/models/__init__.py | 2 +- lmms_eval/models/llava.py | 3 + lmms_eval/models/llava_onevision.py | 38 +- lmms_eval/models/tinyllava.py | 2 + .../video_chatgpt/model/video_chatgpt.py | 1 - lmms_eval/models/vila.py | 2 - lmms_eval/models/xcomposer2_4KHD.py | 1 + lmms_eval/tasks/__init__.py | 2 +- .../_default_template_detailcaps_yaml | 3 + lmms_eval/tasks/detailcaps/detailcaps.yaml | 46 ++ lmms_eval/tasks/detailcaps/utils.py | 198 +++++++ lmms_eval/tasks/gqa_ru/gqa_ru.yaml | 29 + lmms_eval/tasks/gqa_ru/utils.py | 23 + .../llava-in-the-wild/llava-in-the-wild.yaml | 2 +- .../_default_template_wilder_yaml | 2 +- .../tasks/mix_evals/_default_template_yaml | 16 - .../tasks/mix_evals/mix_evals_video2text.yaml | 5 - .../mix_evals_video2text_freeform.yaml | 22 - .../mix_evals/mix_evals_video2text_mc.yaml | 31 -- .../mix_evals_video2text_openended.yaml | 22 - .../mix_evals_video2text_openended_2nd.yaml | 23 - lmms_eval/tasks/mix_evals/utils.py | 284 ---------- lmms_eval/tasks/mlvu/utils.py | 1 - .../mmbench/_default_template_mmbench_ru_yaml | 24 + lmms_eval/tasks/mmbench/mmbench.yaml | 1 + lmms_eval/tasks/mmbench/mmbench_ru_dev.yaml | 10 + lmms_eval/tasks/mmbench/ru_utils.py | 128 +++++ lmms_eval/tasks/mmstar/mmstar.yaml | 37 ++ lmms_eval/tasks/mmstar/utils.py | 120 ++++ lmms_eval/tasks/mmvet/mmvet.yaml | 8 +- lmms_eval/tasks/muirbench/muirbench.yaml | 1 - lmms_eval/tasks/videomme/utils.py | 103 +++- .../tasks/vitatecs/_default_template_yaml | 9 + lmms_eval/tasks/vitatecs/_vitatecs.yaml | 8 + lmms_eval/tasks/vitatecs/utils.py | 225 ++++++++ .../vitatecs/vitatecs_compositionality.yaml | 13 + .../tasks/vitatecs/vitatecs_direction.yaml | 13 + .../tasks/vitatecs/vitatecs_intensity.yaml | 13 + .../tasks/vitatecs/vitatecs_localization.yaml | 13 + .../tasks/vitatecs/vitatecs_sequence.yaml | 13 + lmms_eval/tasks/vitatecs/vitatecs_type.yaml | 13 + pyproject.toml | 4 +- tools/live_bench/example.ipynb | 481 ---------------- tools/live_bench/live_bench/api/live_bench.py | 20 - .../data_generator/live_bench_data.py | 139 ----- .../live_bench/data_generator/qa_generator.py | 522 ------------------ .../live_bench/data_generator/response.py | 12 - .../live_bench/data_generator/score_getter.py | 157 ------ .../live_bench/data_generator/utils/claude.py | 68 --- .../live_bench/data_generator/utils/gemini.py | 37 -- tools/live_bench/live_bench/driver/.gitignore | 1 - .../live_bench/driver/load_driver.py | 71 --- .../live_bench/screen_shoter/screen.py | 30 - .../live_bench/screen_shoter/screen_shoter.py | 141 ----- .../live_bench/websites/load_website.py | 34 -- .../live_bench/live_bench/websites/website.py | 62 --- .../live_bench/websites/website_list.yaml | 78 --- 63 files changed, 1115 insertions(+), 2282 deletions(-) create mode 100644 lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml create mode 100644 lmms_eval/tasks/detailcaps/detailcaps.yaml create mode 100644 lmms_eval/tasks/detailcaps/utils.py create mode 100644 lmms_eval/tasks/gqa_ru/gqa_ru.yaml create mode 100644 lmms_eval/tasks/gqa_ru/utils.py delete mode 100644 lmms_eval/tasks/mix_evals/_default_template_yaml delete mode 100644 lmms_eval/tasks/mix_evals/mix_evals_video2text.yaml delete mode 100644 lmms_eval/tasks/mix_evals/mix_evals_video2text_freeform.yaml delete mode 100644 lmms_eval/tasks/mix_evals/mix_evals_video2text_mc.yaml delete mode 100644 lmms_eval/tasks/mix_evals/mix_evals_video2text_openended.yaml delete mode 100644 lmms_eval/tasks/mix_evals/mix_evals_video2text_openended_2nd.yaml delete mode 100644 lmms_eval/tasks/mix_evals/utils.py create mode 100644 lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml create mode 100644 lmms_eval/tasks/mmbench/mmbench_ru_dev.yaml create mode 100644 lmms_eval/tasks/mmbench/ru_utils.py create mode 100644 lmms_eval/tasks/mmstar/mmstar.yaml create mode 100644 lmms_eval/tasks/mmstar/utils.py create mode 100644 lmms_eval/tasks/vitatecs/_default_template_yaml create mode 100755 lmms_eval/tasks/vitatecs/_vitatecs.yaml create mode 100644 lmms_eval/tasks/vitatecs/utils.py create mode 100644 lmms_eval/tasks/vitatecs/vitatecs_compositionality.yaml create mode 100644 lmms_eval/tasks/vitatecs/vitatecs_direction.yaml create mode 100644 lmms_eval/tasks/vitatecs/vitatecs_intensity.yaml create mode 100644 lmms_eval/tasks/vitatecs/vitatecs_localization.yaml create mode 100644 lmms_eval/tasks/vitatecs/vitatecs_sequence.yaml create mode 100644 lmms_eval/tasks/vitatecs/vitatecs_type.yaml delete mode 100644 tools/live_bench/example.ipynb delete mode 100644 tools/live_bench/live_bench/api/live_bench.py delete mode 100644 tools/live_bench/live_bench/data_generator/live_bench_data.py delete mode 100644 tools/live_bench/live_bench/data_generator/qa_generator.py delete mode 100644 tools/live_bench/live_bench/data_generator/response.py delete mode 100644 tools/live_bench/live_bench/data_generator/score_getter.py delete mode 100644 tools/live_bench/live_bench/data_generator/utils/claude.py delete mode 100644 tools/live_bench/live_bench/data_generator/utils/gemini.py delete mode 100644 tools/live_bench/live_bench/driver/.gitignore delete mode 100644 tools/live_bench/live_bench/driver/load_driver.py delete mode 100644 tools/live_bench/live_bench/screen_shoter/screen.py delete mode 100644 tools/live_bench/live_bench/screen_shoter/screen_shoter.py delete mode 100644 tools/live_bench/live_bench/websites/load_website.py delete mode 100644 tools/live_bench/live_bench/websites/website.py delete mode 100644 tools/live_bench/live_bench/websites/website_list.yaml diff --git a/.gitignore b/.gitignore index 2557ab1bd..edf2efef2 100755 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ temp __pycache__ .ipynb_checkpoints temp +.DS_STORE # IPython profile_default/ ipython_config.py @@ -37,3 +38,4 @@ llava-video/ Video-MME/ VATEX/ lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc +lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc \ No newline at end of file diff --git a/README.md b/README.md index 2bf7a26e3..7be1c6c73 100755 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ ## Annoucement -- [2024-06] 🎬🎬 The `lmms-eval/v0.2` has been upgraded to support video evaluations for video models like LLaVA-NeXT Video and Gemini 1.5 Pro across tasks such as EgoSchema, PerceptionTest, VideoMME, and more. Please refer to the [blog](https://lmms-lab.github.io/posts/lmms-eval-0.2/) for more details +- [2024-07] 👨‍💻👨‍💻 The `lmms-eval/v0.2.1` has been upgraded to support more models, including [LongVA](https://github.com/EvolvingLMMs-Lab/LongVA), [InterVL-2](https://github.com/OpenGVLab/InternVL), [VILA](https://github.com/NVlabs/VILA), and many more evaluation tasks, e.g. [Details Captions](https://github.com/EvolvingLMMs-Lab/lmms-eval/pull/136), [MLVU](https://arxiv.org/abs/2406.04264), [WildVision-Bench](https://huggingface.co/datasets/WildVision/wildvision-arena-data), [VITATECS](https://github.com/lscpku/VITATECS) and [LLaVA-Interleave-Bench](https://llava-vl.github.io/blog/2024-06-16-llava-next-interleave/). + +- [2024-06] 🎬🎬 The `lmms-eval/v0.2.0` has been upgraded to support video evaluations for video models like LLaVA-NeXT Video and Gemini 1.5 Pro across tasks such as EgoSchema, PerceptionTest, VideoMME, and more. Please refer to the [blog](https://lmms-lab.github.io/posts/lmms-eval-0.2/) for more details - [2024-03] 📝📝 We have released the first version of `lmms-eval`, please refer to the [blog](https://lmms-lab.github.io/posts/lmms-eval-0.1/) for more details diff --git a/docs/current_tasks.md b/docs/current_tasks.md index 1622e9602..6e0e52381 100644 --- a/docs/current_tasks.md +++ b/docs/current_tasks.md @@ -50,6 +50,7 @@ - MMMU (mmmu) - MMMU Validation (mmmu_val) - MMMU Test (mmmu_test) +- MMStar (mmstar) - MMUPD (mmupd) - MMUPD Base (mmupd_base) - MMAAD Base (mmaad_base) diff --git a/lmms_eval/__main__.py b/lmms_eval/__main__.py index 996e1e539..ef0e2f1c7 100755 --- a/lmms_eval/__main__.py +++ b/lmms_eval/__main__.py @@ -165,7 +165,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: # reset logger eval_logger.remove() eval_logger.add(sys.stdout, colorize=True, level=args.verbosity) - # eval_logger.add(sys.stderr, level=args.verbosity) eval_logger.info(f"Verbosity set to {args.verbosity}") os.environ["TOKENIZERS_PARALLELISM"] = "false" diff --git a/lmms_eval/api/samplers.py b/lmms_eval/api/samplers.py index f77065e8e..2cecfe224 100755 --- a/lmms_eval/api/samplers.py +++ b/lmms_eval/api/samplers.py @@ -37,7 +37,9 @@ def get_context(self, doc, num_fewshot): + ( str(self.doc_to_target(doc)[0]) if type(self.doc_to_target(doc)) is list - else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)]) + else self.doc_to_target(doc) + if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) + else str(self.doc_to_choice(doc)[self.doc_to_target(doc)]) ) for doc in selected_docs ] diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py index 0e7d05c92..48397a0a2 100755 --- a/lmms_eval/evaluator.py +++ b/lmms_eval/evaluator.py @@ -1,3 +1,5 @@ +import os +import time import random import itertools import json @@ -423,6 +425,12 @@ def evaluate( # Ensure all ranks wait for rank 0 to finish aggregation torch.distributed.barrier() + # Synchronize processes with a temp file in case the evluation metric requires gpus + # TODO: fix barriers' taking up gpu computation + os.makedirs(cli_args.output_path, exist_ok=True) + if os.path.exists(f"{cli_args.output_path}/rank{int(os.environ.get('RANK', 0))}_metric_eval_done.txt"): + os.remove(f"{cli_args.output_path}/rank{int(os.environ.get('RANK', 0))}_metric_eval_done.txt") + if lm.rank == 0: ### Get task ordering for correct sample-wide aggregation group_to_task = {} @@ -623,8 +631,12 @@ def print_tasks(task_hierarchy, task_order, task_version, task_group_alias): } if log_samples: results_dict["samples"] = dict(samples) + else: + results_dict = None - return results_dict + with open(f"{cli_args.output_path}/rank{int(os.environ.get('RANK', 0))}_metric_eval_done.txt", "w") as f: + f.write(f"rank {int(os.environ.get('RANK', 0))} eval done") + while len([file for file in os.listdir(cli_args.output_path) if file.endswith("metric_eval_done.txt")]) < lm._world_size: + time.sleep(1) - else: - return None + return results_dict diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py index 7ba7264c5..d11149a93 100755 --- a/lmms_eval/models/__init__.py +++ b/lmms_eval/models/__init__.py @@ -35,8 +35,8 @@ "mplug_owl_video": "mplug_Owl", "phi3v": "Phi3v", "tinyllava": "TinyLlava", - "llava_hf": "LlavaHf", "llava_onevision": "LlavaOneVision", + "llava_hf": "LlavaHf", "longva": "LongVA", "vila": "VILA", "xcomposer2d5": "XComposer2D5", diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py index 7528e3569..7d6420ba6 100755 --- a/lmms_eval/models/llava.py +++ b/lmms_eval/models/llava.py @@ -58,6 +58,7 @@ def __init__( device_map="cuda:0", conv_template="vicuna_v1", use_cache=True, + tie_weights: bool = True, truncate_context=False, # whether to truncate the context in generation, set it False for LLaVA-1.6 customized_config=None, # ends in json **kwargs, @@ -97,6 +98,8 @@ def __init__( self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args) self._config = self._model.config self.model.eval() + if tie_weights: + self.model.tie_weights() self.truncation = truncation self.batch_size_per_gpu = int(batch_size) diff --git a/lmms_eval/models/llava_onevision.py b/lmms_eval/models/llava_onevision.py index 856aa36cc..4347bc519 100644 --- a/lmms_eval/models/llava_onevision.py +++ b/lmms_eval/models/llava_onevision.py @@ -83,7 +83,7 @@ def __init__( customized_config: Optional[str] = None, # ends in json max_frames_num: Optional[int] = 32, mm_spatial_pool_stride: Optional[int] = 2, - mm_spatial_pool_mode: Optional[str] = "average", + mm_spatial_pool_mode: Optional[str] = "bilinear", token_strategy: Optional[str] = "single", # could be "single" or "multiple", "multiple" denotes adding multiple tokens for each frame video_decode_backend: str = "decord", **kwargs, @@ -183,7 +183,7 @@ def __init__( elif accelerator.num_processes == 1 and device_map == "auto": eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism") self._rank = 0 - self._word_size = 1 + self._world_size = 1 else: eval_logger.info(f"Using single device: {self._device}") @@ -262,6 +262,37 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default. + self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad") + eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") + # if (len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__) and ("image_aspect_ratio" in gen_kwargs.keys()): + # self._config.image_aspect_ratio = gen_kwargs["image_aspect_ratio"] + # eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") + + if type(visual[0]) == PIL.Image.Image: # For image task + image_tensor = process_images(visual, self._image_processor, self._config) + if type(image_tensor) is list: + image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor] + else: + image_tensor = image_tensor.to(dtype=torch.float16, device=self.device) + + task_type = "image" + + elif type(visual[0]) == str: # For video task + image_tensor = [] + try: + if self.video_decode_backend == "decord": + frames = self.load_video(visual, self.max_frames_num) + elif self.video_decode_backend == "pyav": + frames = read_video_pyav(visual[0], num_frm=self.max_frames_num) + frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda() + image_tensor.append(frames) + except Exception as e: + eval_logger.error(f"Error {e} in loading video") + image_tensor = None + + task_type = "video" + # encode, pad, and truncate contexts for this batch if type(doc_to_target) == str: continuation = doc_to_target @@ -319,7 +350,7 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: image_tokens = " ".join(image_tokens) prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts) else: - question = (contexts[0] if isinstance(contexts, list) else contexts) + question = contexts[0] if isinstance(contexts, list) else contexts # This is much safer for llama3, as we now have some object type in it if "llama_3" in self.conv_template: @@ -349,7 +380,6 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: self._config.mm_spatial_pool_stride = self.mm_spatial_pool_stride self._config.mm_spatial_pool_mode = self.mm_spatial_pool_mode - with torch.inference_mode(): outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, **kwargs) loss = outputs["loss"] diff --git a/lmms_eval/models/tinyllava.py b/lmms_eval/models/tinyllava.py index 1cb6d281c..a4335f054 100755 --- a/lmms_eval/models/tinyllava.py +++ b/lmms_eval/models/tinyllava.py @@ -23,7 +23,9 @@ from loguru import logger as eval_logger try: + from tinyllava.model import load_pretrained_model from tinyllava.data import ImagePreprocess, TextPreprocess + from tinyllava.utils.constants import DEFAULT_IMAGE_TOKEN from tinyllava.utils.message import Message except Exception as e: eval_logger.debug("TinyLLaVA_Factory is not installed. Please install TinyLLaVA_Factory to use this model.\nError: %s" % e) diff --git a/lmms_eval/models/video_chatgpt/model/video_chatgpt.py b/lmms_eval/models/video_chatgpt/model/video_chatgpt.py index df6fee4f3..bded27e74 100644 --- a/lmms_eval/models/video_chatgpt/model/video_chatgpt.py +++ b/lmms_eval/models/video_chatgpt/model/video_chatgpt.py @@ -76,7 +76,6 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) if (input_ids.shape[1] != 1 or self.training) and video_spatio_temporal_features is not None: - video_features = self.mm_projector(video_spatio_temporal_features) dummy_video_features = torch.zeros(video_features.shape[1], 1024, device=inputs_embeds.device, dtype=inputs_embeds.dtype) dummy_video_features = self.mm_projector(dummy_video_features) diff --git a/lmms_eval/models/vila.py b/lmms_eval/models/vila.py index 5d48af83f..abec8f765 100755 --- a/lmms_eval/models/vila.py +++ b/lmms_eval/models/vila.py @@ -290,7 +290,6 @@ def generate_until(self, requests) -> List[str]: images = self.load_video(visual, num_video_frames) elif self.video_decode_backend == "pyav": images = read_video_pyav(visual, num_frm=num_video_frames) - video = process_images(images, self.model.image_processor, self.model.config).half().cuda() videos.append(video) @@ -338,7 +337,6 @@ def generate_until(self, requests) -> List[str]: if "num_beams" not in gen_kwargs: gen_kwargs["num_beams"] = 1 - # import pdb;pdb.set_trace() with torch.inference_mode(): output_ids = self.model.generate( input_ids=input_ids, diff --git a/lmms_eval/models/xcomposer2_4KHD.py b/lmms_eval/models/xcomposer2_4KHD.py index e741f637b..6c4f81a77 100644 --- a/lmms_eval/models/xcomposer2_4KHD.py +++ b/lmms_eval/models/xcomposer2_4KHD.py @@ -6,6 +6,7 @@ import torchvision.transforms as transforms from datetime import timedelta + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms diff --git a/lmms_eval/tasks/__init__.py b/lmms_eval/tasks/__init__.py index ce11b8524..8955536c7 100755 --- a/lmms_eval/tasks/__init__.py +++ b/lmms_eval/tasks/__init__.py @@ -73,7 +73,7 @@ def include_task_folder(task_dir: str, register_task: bool = True) -> None: # if (subdirs == [] or subdirs == ["__pycache__"]) and (len(file_list) > 0): for f in file_list: # if "detail" in f: - # import pdb;pdb.set_trace() + # # if "vatex" in f: # print("a") if f.endswith(".yaml"): diff --git a/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml b/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml new file mode 100644 index 000000000..f673aeb7b --- /dev/null +++ b/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml @@ -0,0 +1,3 @@ +model_specific_prompt_kwargs: + default: + prompt: "Describe this image in detail." \ No newline at end of file diff --git a/lmms_eval/tasks/detailcaps/detailcaps.yaml b/lmms_eval/tasks/detailcaps/detailcaps.yaml new file mode 100644 index 000000000..c509f965d --- /dev/null +++ b/lmms_eval/tasks/detailcaps/detailcaps.yaml @@ -0,0 +1,46 @@ +dataset_path: foundation-multimodal-models/DetailCaps-4870 +dataset_kwargs: + token: True +task: "detailcaps" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.detailcaps_doc_to_visual +doc_to_text: !function utils.detailcaps_doc_to_text +doc_to_target: !function utils.detailcaps_doc_to_target +generation_kwargs: + max_new_tokens: 512 + temperature: 0 + top_p: 0 + num_beams: 1 + do_sample: false +process_results: !function utils.detailcaps_process_result +# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results +metric_list: + - metric: detailcaps_CAPTURE + aggregation : !function utils.detailcaps_capture + higher_is_better : true + - metric: detailcaps_Bleu_4 + aggregation : !function utils.detailcaps_bleu4 + higher_is_better : true + - metric: detailcaps_Bleu_3 + aggregation : !function utils.detailcaps_bleu3 + higher_is_better : true + - metric: detailcaps_Bleu_2 + aggregation : !function utils.detailcaps_bleu2 + higher_is_better : true + - metric: detailcaps_Bleu_1 + aggregation : !function utils.detailcaps_bleu1 + higher_is_better : true + - metric: detailcaps_METEOR + aggregation : !function utils.detailcaps_meteor + higher_is_better : true + - metric: detailcaps_ROUGE_L + aggregation : !function utils.detailcaps_rougel + higher_is_better : true + - metric: detailcaps_CIDEr + aggregation : !function utils.detailcaps_cider + higher_is_better : true + +metadata: + - version: 0.0 +include: _default_template_detailcaps_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/detailcaps/utils.py b/lmms_eval/tasks/detailcaps/utils.py new file mode 100644 index 000000000..50c0aba25 --- /dev/null +++ b/lmms_eval/tasks/detailcaps/utils.py @@ -0,0 +1,198 @@ +import collections +import os +import json +from capture_metric.capture import CAPTURE +from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice +from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer +from pycocotools.coco import COCO +import io +from PIL import Image + +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file + +import logging + +eval_logger = logging.getLogger("lmms-eval") + +dir_name = os.path.dirname(os.path.abspath(__file__)) + +detailcaps_METRICS = ["CAPTURE", "Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] + + +def detailcaps_doc_to_visual(doc): + return [Image.open(io.BytesIO(doc["binary"])).convert("RGB")] + + +def detailcaps_doc_to_text(doc, model_specific_prompt_kwargs=None): + # question = "Please carefully observe the image and come up with a caption for the image" + return model_specific_prompt_kwargs["prompt"] + + +def detailcaps_doc_to_target(doc): + references = [ + doc["GT_Caption_GPT4O"], + doc["GT_Caption_GPT4V"], + doc["GT_Caption_Gemini15Pro"], + ] + return references + + +def detailcaps_process_result(doc, result): + """ + Args: + doc: a instance of the eval dataset + results: [pred] + Returns: + a dictionary with key: metric name, value: metric value + """ + + pred = result[0] + # The question id in our dataset is the image file itself + image_id = doc["image"] + + data_dict = {"answer": detailcaps_doc_to_target(doc), "pred": pred, "image_id": image_id} + + return {f"detailcaps_{metric}": data_dict for metric in detailcaps_METRICS} + + +def check_if_context_is_set(expected_context="spawn"): + # 获取默认上下文的名称 + default_context_name = mp.get_context().get_start_method() + + # 检查当前上下文是否与预期的上下文相匹配 + is_set_to_expected = default_context_name == expected_context + + return is_set_to_expected + + +def detailcaps_aggregation_result(results, metric, args=None): + scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr"), (CAPTURE(), "CAPTURE")] + scorers_dict = {s[1]: s for s in scorers} + + stored_results = [] + # In order to make the coco eval tools to successfully create index + # We need at least two dict in the dataset + # 'annotation' and 'images' + # 'annotation' exactly reproduce the original annotation + # 'images' however only need the image id which is contained in the file name + dataset = {"annotations": [], "images": []} + idx = 0 + + for result in results: + stored_results.append({"image_id": result["image_id"], "caption": result["pred"]}) + for a in result["answer"]: + dataset["annotations"].append({"image_id": result["image_id"], "caption": a, "id": idx}) + idx += 1 + dataset["images"].append({"id": result["image_id"]}) + + coco = COCO() + # Manually create index here + coco.dataset = dataset + coco.createIndex() + + detailcaps_result = coco.loadRes(stored_results) + detailcaps_eval = COCOEvalCap(coco, detailcaps_result) + + imgIds = detailcaps_eval.params["image_id"] + gts = {} + res = {} + for imgId in imgIds: + gts[imgId] = detailcaps_eval.coco.imgToAnns[imgId] + res[imgId] = detailcaps_eval.cocoRes.imgToAnns[imgId] + + eval_logger.info("tokenization...") + tokenizer = PTBTokenizer() + + if metric == "CAPTURE": + reorg_gts, reorg_res = collections.defaultdict(list), collections.defaultdict(list) + for _, samples in gts.items(): + for sample in samples: + reorg_gts[sample["image_id"]].append(sample["caption"]) + for _, samples in res.items(): + for sample in samples: + reorg_res[sample["image_id"]].append(sample["caption"]) + gts, res = reorg_gts, reorg_res + else: + gts = tokenizer.tokenize(gts) + res = tokenizer.tokenize(res) + + eval_logger.info(f"Computing {metric} scores...") + + # if int(os.environ.get("RANK", 0)) == 0: + # from IPython import embed; embed() + # else: + # import time; time.sleep(1200) + + score, scores = scorers_dict[metric][0].compute_score(gts, res) + # When metric is one of the Bleu, score will be a list + if type(score) == list: + n = int(metric.split("_")[-1]) + score = score[n - 1] + + path = generate_submission_file(f"detailcaps_val_{metric}_scores.json", args) + eval_logger.info("Storing prediction that can be submitted to the server ...") + with open(path, "w") as f: + json.dump(stored_results, f, indent=4) + eval_logger.info(f"Your result has been saved to {path}.") + + return score + + +def detailcaps_bleu4(results, args=None): + return detailcaps_aggregation_result(results, "Bleu_4", args) + + +def detailcaps_bleu3(results, args=None): + return detailcaps_aggregation_result(results, "Bleu_3", args) + + +def detailcaps_bleu2(results, args=None): + return detailcaps_aggregation_result(results, "Bleu_2", args) + + +def detailcaps_bleu1(results, args=None): + return detailcaps_aggregation_result(results, "Bleu_1", args) + + +def detailcaps_meteor(results, args=None): + return detailcaps_aggregation_result(results, "METEOR", args) + + +def detailcaps_rougel(results, args=None): + return detailcaps_aggregation_result(results, "ROUGE_L", args) + + +def detailcaps_cider(results, args=None): + return detailcaps_aggregation_result(results, "CIDEr", args) + + +def detailcaps_spice(results, args=None): + return detailcaps_aggregation_result(results, "SPICE", args) + + +def detailcaps_capture(results, args=None): + return detailcaps_aggregation_result(results, "CAPTURE", args) + + +def detailcaps_test_process_result(doc, result): + """ + Args: + doc: a instance of the eval dataset + results: [pred] + Returns: + a dictionary with key: metric name (in this case detailcaps_passthrough), value: metric value + """ + return {"detailcaps_passthrough": {"pred": result[0], "image_id": doc["image_id"]}} + + +def detailcaps_test_aggregation_result(results, args=None): + stored_results = [] + for result in results: + stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]}) + + path = generate_submission_file("detailcaps_captions_detailcaps_test_alg_results.json", args) + eval_logger.info("Storing prediction that can be submitted to the server ...") + with open(path, "w") as f: + json.dump(stored_results, f, indent=4) + + eval_logger.info(f"Your test result has been stored in {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.") diff --git a/lmms_eval/tasks/gqa_ru/gqa_ru.yaml b/lmms_eval/tasks/gqa_ru/gqa_ru.yaml new file mode 100644 index 000000000..2a3d10972 --- /dev/null +++ b/lmms_eval/tasks/gqa_ru/gqa_ru.yaml @@ -0,0 +1,29 @@ +dataset_path: deepvk/GQA-ru +dataset_name: testdev_balanced_instructions +dataset_kwargs: + token: True +task: "gqa-ru" +test_split: testdev +output_type: generate_until +doc_to_visual: !function utils.gqa_doc_to_visual +doc_to_text: !function utils.gqa_doc_to_text +doc_to_target: "answer" +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + - version: 0.0 + +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nОтветь одним словом." \ No newline at end of file diff --git a/lmms_eval/tasks/gqa_ru/utils.py b/lmms_eval/tasks/gqa_ru/utils.py new file mode 100644 index 000000000..9c1acb9ce --- /dev/null +++ b/lmms_eval/tasks/gqa_ru/utils.py @@ -0,0 +1,23 @@ +from datasets import load_dataset + +GQA_RAW_IMAGE_DATASET = None +GQA_ID2IMAGE = None + + +def gqa_doc_to_visual(doc): + global GQA_RAW_IMAGE_DATASET + global GQA_ID2IMAGE + if GQA_RAW_IMAGE_DATASET is None: + GQA_RAW_IMAGE_DATASET = load_dataset("deepvk/GQA-ru", "testdev_balanced_images", split="testdev", token=True) + GQA_ID2IMAGE = {} + for row in GQA_RAW_IMAGE_DATASET: + GQA_ID2IMAGE[row["id"]] = row["image"].convert("RGB") + image = GQA_ID2IMAGE[doc["imageId"]] + return [image] + + +def gqa_doc_to_text(doc, model_specific_prompt_kwargs): + question = doc["question"] + pre_prompt = model_specific_prompt_kwargs["pre_prompt"] + post_prompt = model_specific_prompt_kwargs["post_prompt"] + return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml b/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml index 02e846c37..c0db28e18 100755 --- a/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml +++ b/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml @@ -11,7 +11,7 @@ generation_kwargs: until: - "ASSISTANT:" image_aspect_ratio: original - max_new_tokens: 32768 + max_new_tokens: 4096 temperature: 0 top_p: 1.0 num_beams: 1 diff --git a/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml b/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml index 356df525f..2484f795c 100644 --- a/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml +++ b/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml @@ -4,7 +4,7 @@ doc_to_text: !function utils.llava_doc_to_text doc_to_target: "gpt4v_answer" generation_kwargs: max_new_tokens: 4096 - temperature: 0.7 + temperature: 0 top_p: 1.0 num_beams: 1 do_sample: false diff --git a/lmms_eval/tasks/mix_evals/_default_template_yaml b/lmms_eval/tasks/mix_evals/_default_template_yaml deleted file mode 100644 index ce26b0ea1..000000000 --- a/lmms_eval/tasks/mix_evals/_default_template_yaml +++ /dev/null @@ -1,16 +0,0 @@ -dataset_path: lmms-lab/MixEvals_Video2Text -dataset_kwargs: - token: True - video: True - cache_dir: mix_evals_video2text -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" - gpt4v: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "" -metadata: - modality: video - version: 0.0 - gpt_eval_model_name: "gpt-3.5-turbo" \ No newline at end of file diff --git a/lmms_eval/tasks/mix_evals/mix_evals_video2text.yaml b/lmms_eval/tasks/mix_evals/mix_evals_video2text.yaml deleted file mode 100644 index ed0a517c0..000000000 --- a/lmms_eval/tasks/mix_evals/mix_evals_video2text.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group: mix_evals_video2text -task: -- mix_evals_video2text_openconv -- mix_evals_video2text_mc -- mix_evals_video2text_freeform \ No newline at end of file diff --git a/lmms_eval/tasks/mix_evals/mix_evals_video2text_freeform.yaml b/lmms_eval/tasks/mix_evals/mix_evals_video2text_freeform.yaml deleted file mode 100644 index e8ec9b4ad..000000000 --- a/lmms_eval/tasks/mix_evals/mix_evals_video2text_freeform.yaml +++ /dev/null @@ -1,22 +0,0 @@ -dataset_name: "video2text_closeended_free-form" -task: "mix_evals_video2text_freeform" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mix_evals_video2text_doc_to_visual -doc_to_text: !function utils.mix_evals_video2text_doc_to_text -doc_to_target: "{{target}}" -process_results: !function utils.mix_evals_video2text_process_results_freeform -metric_list: - - metric: gpt_eval - aggregation: !function utils.mix_evals_video2text_gpt_eval - higher_is_better: true - -include: _default_template_yaml - -model_specific_prompt_kwargs: - default: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "Answer the question using a single word or phrase." - gpt4v: - pre_prompt: "These are frames from a video. Please answer the following questions about the video with a short phrase." - post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/mix_evals/mix_evals_video2text_mc.yaml b/lmms_eval/tasks/mix_evals/mix_evals_video2text_mc.yaml deleted file mode 100644 index d04dabf42..000000000 --- a/lmms_eval/tasks/mix_evals/mix_evals_video2text_mc.yaml +++ /dev/null @@ -1,31 +0,0 @@ -include: _default_template_yaml -dataset_name: "video2text_closeended_multiple-choice" -task: "mix_evals_video2text_mc" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mix_evals_video2text_doc_to_visual -doc_to_text: !function utils.mix_evals_video2text_doc_to_text -doc_to_target: "{{target}}" - -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - -filter_list: - - name: "flexible-extract" - filter: - - function: !function utils.MultiChoiceRegexFilter - group_select: 0 - ignore_case: true - ignore_punctuation: true - -model_specific_prompt_kwargs: - default: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "Answer with the option's letter from the given choices directly." - gpt4v: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mix_evals/mix_evals_video2text_openended.yaml b/lmms_eval/tasks/mix_evals/mix_evals_video2text_openended.yaml deleted file mode 100644 index eb3fca3db..000000000 --- a/lmms_eval/tasks/mix_evals/mix_evals_video2text_openended.yaml +++ /dev/null @@ -1,22 +0,0 @@ -include: _default_template_yaml -dataset_name: "video2text_openended" -task: "mix_evals_video2text_openconv" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mix_evals_video2text_doc_to_visual -doc_to_text: !function utils.mix_evals_video2text_doc_to_text_open_convs -doc_to_target: "" -process_results: !function utils.mix_evals_video2text_process_results_open_convs - -metric_list: - - metric: submission - aggregation: !function utils.mix_evals_video2text_aggregate_gen - higher_is_better: true - -model_specific_prompt_kwargs: - default: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "" - gpt4v: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "" diff --git a/lmms_eval/tasks/mix_evals/mix_evals_video2text_openended_2nd.yaml b/lmms_eval/tasks/mix_evals/mix_evals_video2text_openended_2nd.yaml deleted file mode 100644 index e8c0f1fe7..000000000 --- a/lmms_eval/tasks/mix_evals/mix_evals_video2text_openended_2nd.yaml +++ /dev/null @@ -1,23 +0,0 @@ -include: _default_template_yaml -dataset_path: lmms-lab/MixEvals_Video2Text_OpenEnded_2nd -dataset_name: "video2text_openended" -task: "mix_evals_video2text_openconv_2nd" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mix_evals_video2text_doc_to_visual -doc_to_text: !function utils.mix_evals_video2text_doc_to_text_open_convs -doc_to_target: "" -process_results: !function utils.mix_evals_video2text_process_results_open_convs - -metric_list: - - metric: submission - aggregation: !function utils.mix_evals_video2text_aggregate_gen - higher_is_better: true - -model_specific_prompt_kwargs: - default: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "" - gpt4v: - pre_prompt: "These are frames from a video. Please answer the following questions about the video." - post_prompt: "" diff --git a/lmms_eval/tasks/mix_evals/utils.py b/lmms_eval/tasks/mix_evals/utils.py deleted file mode 100644 index 6c3e6e6c3..000000000 --- a/lmms_eval/tasks/mix_evals/utils.py +++ /dev/null @@ -1,284 +0,0 @@ -import os -import re -import sys -import datetime -import lmms_eval.tasks._task_utils.file_utils as file_utils -from lmms_eval.filters.extraction import ExtendedRegexFilter -import json -import yaml -from pathlib import Path -import requests -import time -from loguru import logger as eval_logger - -with open(Path(__file__).parent / "_default_template_yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -NUM_SECONDS_TO_SLEEP = 5 -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - -eval_prompt = """You are an AI assistant who will help me to evaluate the quality of a model response to a few candidate ground truth answers. - -Some criterion -- Response that perfectly reflect the meaning of the ground truth: 1 point -- Response that reflect none of the key points in the ground truth: 0 point -- Some part in the response are correct but some parts in the ground truth are not mentioned in the response: 0.5 point -- Some part in the response are correct but other parts in the response are not mentioned in the ground truth: 0.5 point - -Here're some examples about the scoring criterion and format: -model response: Steam Cleaning Services -ground truth: ["steam clean", "steam clean", "cleaning", "car", "steam clean"], -Point: 1 - -model response: A cowboy action shooter. -ground truth: ["man"] -Point: 1 - -model response: I'm sorry, but I can't assist with that request. -ground truth: ["quality"] -Point: 0 - -Let's begin this task: -model response: {model_response} -ground truth: {ground_truth} -Point:""" - - -def get_eval(model_response: str, ground_truth: str, max_tokens: int, retries: int = 5): - global headers - content = eval_prompt.format(model_response=model_response, ground_truth=ground_truth) - - messages = [ - {"role": "user", "content": content}, - ] - - payload = { - "model": GPT_EVAL_MODEL_NAME, - "messages": messages, - "temperature": 0.2, - "max_tokens": max_tokens, - } - - for attempt in range(retries): - try: - response = requests.post(API_URL, headers=headers, json=payload, timeout=60) - response.raise_for_status() - response_data = response.json() - - content = response_data["choices"][0]["message"]["content"].strip() - if content != "": - return content, response_data["model"] - break # If successful, break out of the loop - - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}") - if attempt < retries: # If we have retries left, sleep and then continue to next attempt - time.sleep(NUM_SECONDS_TO_SLEEP) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {e}") - return "", "" - return "", "" - - -# A bit ugly here -# But the idea is that we will unzip all the zip files -# To HF HOME cache dir -# And load it here -HF_HOME = os.environ["HF_HOME"] -cache_dir = config["dataset_kwargs"]["cache_dir"] -cache_dir = os.path.join(HF_HOME, cache_dir) -cache_dir = os.path.join(cache_dir) - - -# Pass in video path here -# Can only work correctly with video llm -def mix_evals_video2text_doc_to_visual(doc): - video_path = doc["video_path"] - video_path = os.path.join(cache_dir, video_path) - if os.path.exists(video_path): - video_path = video_path - elif os.path.exists(video_path.replace("mp4", "MP4")): - video_path = video_path.replace("mp4", "MP4") - else: - sys.exit(f"video path:{video_path} does not exist, please check") - return [video_path] - - -# This is the place where you format your question -def mix_evals_video2text_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - - user_prompt = doc["prompt"] - - if "options" in doc: - option_prompt = "Here are the options:\n" - for idx, option in enumerate(doc["options"]): - char_idx = chr(ord("A") + idx) - option = option.strip() - option_prompt += f"{char_idx}. {option}\n" - - option_prompt = option_prompt.rstrip("\n") - user_prompt = f"{user_prompt}\n{option_prompt}" - - if pre_prompt: - user_prompt = f"{pre_prompt}\n{user_prompt}" - - if post_prompt: - user_prompt = f"{user_prompt}\n{post_prompt}" - return user_prompt - - -OPEN_CONVS_PROMPT = """{PRE} -{FIRST} -{POST} -""" - - -def mix_evals_video2text_doc_to_text_open_convs(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - - filtered_first_turn = re.sub(r"", "", doc["first_turn_user_prompt"]) - return OPEN_CONVS_PROMPT.format( - PRE=pre_prompt, - POST=post_prompt, - FIRST=filtered_first_turn, - ) - - -MODEL_CONVS_PROMPT = """{FIRST} -{MODEL_RESPONSE} -{PRE} -{SECOND} -{POST} -""" - - -def mix_evals_video2text_doc_to_text_open_2nd_convs(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - - return MODEL_CONVS_PROMPT.format( - PRE=pre_prompt, - POST=post_prompt, - FIRST=doc["first_turn_user_prompt"], - SECOND=doc["second_turn_user_prompt"], - MODEL_RESPONSE=doc["model_response"], - ) - - -def mix_evals_video2text_process_results_open_convs(doc, result): - pred = result[0] - return {"submission": {"pred": pred, "question_idx": doc["question_index"], "first_turn_video_caption": doc["first_turn_video_caption"], "target": ""}} - - -def mix_evals_video2text_process_results_freeform(doc, result): - pred = result[0] - ground_truth_str = ", ".join([f'"{gt}"' for gt in doc["target"]]) - ground_truth_str = f"[{ground_truth_str}]" - content = eval_prompt.format(model_response=pred, ground_truth=ground_truth_str) - eval_answer, model_name = get_eval(model_response=pred, ground_truth=ground_truth_str, max_tokens=1024) - return { - "submission": {"pred": pred, "question_idx": doc["question_index"], "target": doc["target"], "eval_answer": eval_answer, "gpt_prompt": content}, - "gpt_eval": {"pred": pred, "question_idx": doc["question_index"], "target": doc["target"], "eval_answer": eval_answer, "gpt_prompt": content}, - } - - -def mix_evals_video2text_aggregate_submissions(results, args, task): - now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - submission_file_name = f"mix_evals_video2text_{task}-{now_date_time}.json" - path = file_utils.generate_submission_file(submission_file_name, args) - with open(path, "w") as f: - json.dump(results, f) - eval_logger.info(f"Submission file saved to {path}") - - -def mix_evals_video2text_gpt_eval(results, args): - score = 0 - for result in results: - eval_answer = result["eval_answer"] - eval_score = re.search(r"([0-9.]+)", eval_answer).group(1) - try: - eval_score = float(eval_score) - except Exception as e: - eval_logger.error(f"Error parsing eval_score: {e}") - eval_score = 0.0 - score += eval_score - - return score / len(results) - - -# Factory into different aggregate -def mix_evals_video2text_aggregate_gen(results, args): - mix_evals_video2text_aggregate_submissions(results, args, "OpenConvs") - - -class MultiChoiceRegexFilter(ExtendedRegexFilter): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def apply(self, resps, docs): - filtered_resps = [] - - for r, doc in zip(resps, docs): - # Regex to directly extract the option letter from the model response - option_letter_regex = re.compile(r"\b([A-Z])\.\s+([^\n]*)") - - # Process each response - filtered = [] - for resp in r: - # Try to match the option letter at the start of the response - match = option_letter_regex.match(resp) - if match: - # If a match is found, append the matched letter - filtered.append(match.group(1)) - else: - # If no match, return the original response - filtered.append(resp) - - # Assuming we need the first response that matches or the original response - filtered_resps.append(filtered[0]) - - return filtered_resps diff --git a/lmms_eval/tasks/mlvu/utils.py b/lmms_eval/tasks/mlvu/utils.py index 046d5423a..9829ee8af 100644 --- a/lmms_eval/tasks/mlvu/utils.py +++ b/lmms_eval/tasks/mlvu/utils.py @@ -29,7 +29,6 @@ def mlvu_doc_to_visual(doc): - cache_dir = os.path.join(base_cache_dir, cache_name) video_path = doc["video_name"] video_path = os.path.join(cache_dir, video_path) diff --git a/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml b/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml new file mode 100644 index 000000000..993cd52ad --- /dev/null +++ b/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml @@ -0,0 +1,24 @@ +dataset_path: deepvk/MMBench-ru +dataset_kwargs: + token: True +doc_to_target: "answer" +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nВыбери правильный вариант ответа буквой." +doc_to_visual: !function ru_utils.mmbench_doc_to_visual +doc_to_text: !function ru_utils.mmbench_doc_to_text +doc_to_target: "answer" +process_results: !function ru_utils.mmbench_process_results +model_specific_generation_kwargs: + llava: + image_aspect_ratio: original +output_type: generate_until +generation_kwargs: + until: + - "ASSISTANT:" + max_new_tokens: 1024 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false diff --git a/lmms_eval/tasks/mmbench/mmbench.yaml b/lmms_eval/tasks/mmbench/mmbench.yaml index 821065eea..f2546aa5f 100755 --- a/lmms_eval/tasks/mmbench/mmbench.yaml +++ b/lmms_eval/tasks/mmbench/mmbench.yaml @@ -5,6 +5,7 @@ task: - mmbench_cn_dev - mmbench_cn_test - mmbench_cn_cc + - mmbench_ru_dev metadata: version: 0.0 sys_prompt: "There are several options:" diff --git a/lmms_eval/tasks/mmbench/mmbench_ru_dev.yaml b/lmms_eval/tasks/mmbench/mmbench_ru_dev.yaml new file mode 100644 index 000000000..46407ae46 --- /dev/null +++ b/lmms_eval/tasks/mmbench/mmbench_ru_dev.yaml @@ -0,0 +1,10 @@ +task: "mmbench_ru_dev" +test_split: dev +include: _default_template_mmbench_ru_yaml +metric_list: + - metric: gpt_eval_score + aggregation: !function ru_utils.mmbench_aggregate_dev_results_eval + higher_is_better: true + - metric: submission + aggregation: !function ru_utils.mmbench_aggregate_dev_results_submission + higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmbench/ru_utils.py b/lmms_eval/tasks/mmbench/ru_utils.py new file mode 100644 index 000000000..3ff515c7c --- /dev/null +++ b/lmms_eval/tasks/mmbench/ru_utils.py @@ -0,0 +1,128 @@ +import yaml +import os +from pathlib import Path +import pandas as pd +import json + +from loguru import logger as eval_logger +from lmms_eval.tasks.mmbench.mmbench_evals import MMBench_Evaluator +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file + +with open(Path(__file__).parent / "mmbench.yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + +GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] +API_TYPE = os.getenv("API_TYPE", "openai") + +if API_TYPE == "openai": + API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") + API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") +elif API_TYPE == "azure": + API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") + API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") +else: + API_URL = "YOUR_API_URL" + API_KEY = "YOUR_API_KEY" + + +mmbench_evaluator = MMBench_Evaluator(sys_prompt=config["metadata"]["sys_prompt"], API_KEY=API_KEY, API_URL=API_URL, model_version=GPT_EVAL_MODEL_NAME) + + +def mmbench_doc_to_visual(doc): + return [doc["image"].convert("RGB")] + + +def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): + option_candidate = ["A", "B", "C", "D", "E"] + options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) + + data = { + # "img": doc["image"], + "question": doc["question"], + "answer": doc.get("answer", None), + "options": options_prompt, + "category": doc["category"], + "L2-category": doc["l2-category"], + "options_dict": options_dict, + "index": doc["index"], + "hint": doc["hint"], + "source": doc["source"], + "split": doc["split"], + } + + query_prompt = f"{data['hint']} {data['question']} {data['options']}" if pd.notna(data["hint"]) and data["hint"] != "nan" else f"{data['question']} {data['options']}" + + if model_specific_prompt_kwargs: + query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" + + return query_prompt + + +def mmbench_process_results(doc, results): + model_response = results[0].strip() + data = { + "gpt_eval_score": { + "index": doc["index"], + "question": doc["question"], + "answer": doc["answer"], + "prediction": model_response, + "hint": doc["hint"], + "source": doc["source"], + "split": doc["split"], + "category": doc["category"], + "L2-category": doc["l2-category"], + }, + "submission": { + "index": doc["index"], + "question": doc["question"], + "answer": doc["answer"], + "prediction": model_response, + "hint": doc["hint"], + "source": doc["source"], + "split": doc["split"], + "category": doc["category"], + "L2-category": doc["l2-category"], + }, + } + option_candidate = ["A", "B", "C", "D", "E"] + for c in option_candidate: + data["submission"][c] = doc.get(c, "nan") + data["gpt_eval_score"][c] = doc.get(c, "nan") + return data + + +def mmbench_aggregate_dev_results_eval(results, args): + print(f"============= MMBench-RU(Dev) Detailed Results =============") + overall_acc, category_acc, l2_category_acc = mmbench_evaluator.eval_result(results, eval_method="openai") + file = generate_submission_file("mmbench_ru_dev_results.json", args) + details_info = { + "overall_acc": overall_acc, + "category_acc": category_acc, + "l2_category_acc": l2_category_acc, + } + with open(file, "w") as f: + json.dump(details_info, f) + return overall_acc * 100 + + +def mmbench_aggregate_dev_results_submission(results, args): + df = pd.DataFrame(results) + excel_write_path = generate_submission_file("mmbench_ru_dev_results.xlsx", args) + with pd.ExcelWriter(excel_write_path) as writer: + df.to_excel(writer, index=False) + eval_logger.info(f"Saved results to {excel_write_path}") + + +def mmbench_aggregate_test_results(results, args): + df = pd.DataFrame(results) + excel_write_path = generate_submission_file("mmbench_ru_test_results.xlsx", args) + with pd.ExcelWriter(excel_write_path) as writer: + df.to_excel(writer, index=False) + eval_logger.info(f"Saved results to {excel_write_path}") diff --git a/lmms_eval/tasks/mmstar/mmstar.yaml b/lmms_eval/tasks/mmstar/mmstar.yaml new file mode 100644 index 000000000..845978460 --- /dev/null +++ b/lmms_eval/tasks/mmstar/mmstar.yaml @@ -0,0 +1,37 @@ +dataset_path: Lin-Chen/MMStar +dataset_kwargs: + token: True +task: "mmstar" +test_split: val +output_type: generate_until +doc_to_visual: !function utils.mmstar_doc_to_visual +doc_to_text: !function utils.mmstar_doc_to_text +doc_to_target: "answer" +# The return value of process_results will be used by metrics +process_results: !function utils.mmstar_process_results +# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results +metric_list: + - metric: coarse perception + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true + - metric: fine-grained perception + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true + - metric: instance reasoning + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true + - metric: logical reasoning + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true + - metric: science & technology + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true + - metric: math + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nAnswer with the option's letter from the given choices directly" +metadata: + - version: 0.0 diff --git a/lmms_eval/tasks/mmstar/utils.py b/lmms_eval/tasks/mmstar/utils.py new file mode 100644 index 000000000..66933f299 --- /dev/null +++ b/lmms_eval/tasks/mmstar/utils.py @@ -0,0 +1,120 @@ +from collections import defaultdict +import os +import datetime +import json +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file + + +from loguru import logger as eval_logger + +dir_name = os.path.dirname(os.path.abspath(__file__)) + +eval_type_dict = { + "coarse perception" : [ + "image scene and topic", + "image style & quality", + "image emotion" + ], + "fine-grained perception" : [ + "object counting", + "recognition", + "localization" + ], + "instance reasoning" : [ + "single-instance reasoning", + "cross-instance attribute reasoning", + "cross-instance relation reasoning" + ], + "logical reasoning" : [ + "code & sequence reasoning", + "diagram reasoning", + "common reasoning" + ], + "science & technology" : [ + "biology & chemistry & physics", + "electronics & energy & mechanical eng.", + "geography & earth science & agriculture" + ], + "math" : [ + "geometry", + "numeric commonsense and calculation", + "statistical reasoning" + ] +} + + +replace_prompt = " Please answer yes or no." + + +def mmstar_doc_to_visual(doc): + return [doc["image"].convert("RGB")] + + +def mmstar_doc_to_text(doc, model_specific_prompt_kwargs=None): + question = doc["question"].strip() + if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": + question = question.replace(replace_prompt, "") + question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" + if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": + question = question.replace(replace_prompt, "") + question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + return question + + +def exact_match(pred, gt): + """Brought from MMStar""" + answer = gt.lower().strip().replace('\n', ' ') + predict = pred.lower().strip().replace('\n', ' ') + try: + if answer == predict[0]: + return 1.0 + elif predict[0] == '(' and answer == predict[1]: + return 1.0 + elif predict[0:7] == 'option ' and answer == predict[7]: + return 1.0 + elif predict[0:14] == 'the answer is ' and answer == predict[14]: + return 1.0 + except Exception as e: + return 0.0 + return 0.0 + + +def mmstar_process_results(doc, results): + """ + Args: + doc: a instance of the eval dataset + results: [pred] + Returns: + a dictionary with key: metric name, value: metric value + """ + pred = results[0] + gt = doc["answer"] + + score = exact_match(pred, gt) + category = doc["category"] + l2_category = doc["l2_category"] + return {category: {"question_id": doc["index"], "l2_category": l2_category, "score": score}} + + +def mmstar_aggregate_results(results): + """ + Args: + results: a list of values returned by process_results + Returns: + A score + """ + l2_category_scores = defaultdict(list) + for result in results: + score = result["score"] + l2_category = result["l2_category"] + l2_category_scores[l2_category].append(score) + + l2_category_avg_score = {} + for l2_category, scores in l2_category_scores.items(): + avg_score = sum(scores) / len(scores) + l2_category_avg_score[l2_category] = avg_score + eval_logger.info(f"{l2_category}: {avg_score:.2f}") + + avg_score = sum(l2_category_avg_score.values()) / len(l2_category_avg_score) + return avg_score + \ No newline at end of file diff --git a/lmms_eval/tasks/mmvet/mmvet.yaml b/lmms_eval/tasks/mmvet/mmvet.yaml index 30c1907aa..827856db6 100755 --- a/lmms_eval/tasks/mmvet/mmvet.yaml +++ b/lmms_eval/tasks/mmvet/mmvet.yaml @@ -8,10 +8,8 @@ doc_to_visual: !function utils.mmvet_doc_to_visual doc_to_text: !function utils.doc_to_text # Such that {{question}} will be replaced by doc["question"] doc_to_target: "{{answer}}" generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 32768 - temperature: 0 + max_new_tokens: 1024 + temperature: 0.2 top_p: 1.0 num_beams: 1 do_sample: false @@ -25,5 +23,5 @@ metadata: gpt_eval_model_name: "gpt-4-0613" model_specific_prompt_kwargs: default: - pre_prompt: "Please think step by step and try to provide best answer to the following question: \n\n" + pre_prompt: "First please perform reasoning, and think step by step to provide best answer to the following question: \n\n" post_prompt: "" diff --git a/lmms_eval/tasks/muirbench/muirbench.yaml b/lmms_eval/tasks/muirbench/muirbench.yaml index 43b8ab7cc..896314772 100644 --- a/lmms_eval/tasks/muirbench/muirbench.yaml +++ b/lmms_eval/tasks/muirbench/muirbench.yaml @@ -1,4 +1,3 @@ - dataset_path: MUIRBENCH/MUIRBENCH task: "muirbench" dataset_kwargs: diff --git a/lmms_eval/tasks/videomme/utils.py b/lmms_eval/tasks/videomme/utils.py index 401b02a04..c9ec7aa6e 100644 --- a/lmms_eval/tasks/videomme/utils.py +++ b/lmms_eval/tasks/videomme/utils.py @@ -133,6 +133,43 @@ def extract_subtitles(video_path, subtitle_path): return subtitle_frames, total_frame +def parse_subtitle_time(time_str): + h, m, s_ms = time_str.split(':') + s, ms = s_ms.split(',') + return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000 + +def load_subtitles(subtitle_path): + subtitles = {} + with open(subtitle_path, 'r', encoding='utf-8') as file: + content = file.read().split('\n\n') + for section in content: + if section.strip(): + lines = section.split('\n') + if len(lines) >= 3: + time_range = lines[1].split(' --> ') + start_time = parse_subtitle_time(time_range[0]) + end_time = parse_subtitle_time(time_range[1]) + text = ' '.join(line for line in lines[2:]) + subtitles[(start_time, end_time)] = text + return subtitles + +def convert_time_to_frame(time_in_seconds, fps): + return int(time_in_seconds * fps) + +def extract_subtitles(video_path, subtitle_path): + video = cv2.VideoCapture(video_path) + fps = video.get(cv2.CAP_PROP_FPS) + total_frame=int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + subtitles = load_subtitles(subtitle_path) + + subtitle_frames = [] + for (start_time, end_time), text in subtitles.items(): + start_frame = convert_time_to_frame(start_time, fps) + end_frame = convert_time_to_frame(end_time, fps) + subtitle_frames.append((start_frame, end_frame, text)) + + return subtitle_frames,total_frame + def videomme_doc_to_visual(doc): cache_dir = os.path.join(base_cache_dir, cache_name) video_path = doc["videoID"] + ".mp4" @@ -149,7 +186,71 @@ def videomme_doc_to_visual(doc): def videomme_doc_to_text(doc, model_specific_prompt_kwargs=None): - option_prompt = "Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." + option_prompt="Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." + question = doc["question"] + option = str(doc["options"]) + question = question + "\n" + option + full_prompt=option_prompt+"\n"+question+"\n"+"The best answer is:" + return full_prompt +# Frames + Subs +# This video's subtitles are listed below: +# 【subtitles】 + +# Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option. +# 【question】 +# The best answer is: +# Frames / Frames + Audio +# Select the best answer to the following multiple-choice question based on the video. Respond with only the letter (A, B, C, or D) of the correct option. +# 【question】 +# The best answer is: + +def videomme_doc_to_text_subtitle(doc, model_specific_prompt_kwargs=None): + cache_dir = os.path.join(base_cache_dir, cache_name) + video_path = doc["videoID"] + ".mp4" + subtitle_path=os.path.join(cache_dir,"subtitle",doc["videoID"]+".srt") + video_path = os.path.join(cache_dir, video_path) + if os.path.exists(subtitle_path): #Denote have subtitle + subtitle=open(subtitle_path).readlines() + else: + subtitle="" + subtitles_prompt="This video's subtitles are listed below: \n" + if subtitle=="": + subtitle="No subtitles available" + else: + if "gemini_api_flag" in model_specific_prompt_kwargs: #specific for gemini_api + if model_specific_prompt_kwargs['gemini_api_flag']=="full subtitle": + textlist=[] + for ele in subtitle: + pattern = r'(.*?)' + matches = re.findall(pattern, ele) + if matches: + textlist.append(matches[0]) + subtitle_text="\n".join(textlist) + else: + if "frame_num" in model_specific_prompt_kwargs: + frame_num=model_specific_prompt_kwargs['frame_num'] + subtitle_by_frame,total_frame=extract_subtitles(video_path,subtitle_path) + uniform_sampled_frames = np.linspace(0, total_frame - 1, frame_num, dtype=int).tolist() + + subtitle_by_frame_idx=[] + for frame_idx in uniform_sampled_frames: + for idx,title in enumerate(subtitle_by_frame): + if frame_idx=title[0]: + subtitle_by_frame_idx.append(idx) + subtitle_by_frame_idx=list(set(subtitle_by_frame_idx)) + + textlist=[] + for idx in subtitle_by_frame_idx: + pattern = r'(.*?)' + raw_text=re.findall(pattern, subtitle_by_frame[idx][2]) + try: + textlist.append(raw_text[0]) + except: + continue + subtitle_text="\n".join(textlist) + subtitle=subtitle_text + + option_prompt="Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." question = doc["question"] option = str(doc["options"]) # option = "\n".join([f"{opt}" for i, opt in enumerate(doc["options"])]) diff --git a/lmms_eval/tasks/vitatecs/_default_template_yaml b/lmms_eval/tasks/vitatecs/_default_template_yaml new file mode 100644 index 000000000..e93ce7d5e --- /dev/null +++ b/lmms_eval/tasks/vitatecs/_default_template_yaml @@ -0,0 +1,9 @@ +dataset_path: lscpku/VITATECS +dataset_kwargs: + token: True + video: True + cache_dir: vitatecs +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nPlease response with a single letter (A or B):" \ No newline at end of file diff --git a/lmms_eval/tasks/vitatecs/_vitatecs.yaml b/lmms_eval/tasks/vitatecs/_vitatecs.yaml new file mode 100755 index 000000000..07677c284 --- /dev/null +++ b/lmms_eval/tasks/vitatecs/_vitatecs.yaml @@ -0,0 +1,8 @@ +group: vitatecs +task: +- vitatecs_direction +- vitatecs_intensity +- vitatecs_sequence +- vitatecs_compositionality +- vitatecs_localization +- vitatecs_type diff --git a/lmms_eval/tasks/vitatecs/utils.py b/lmms_eval/tasks/vitatecs/utils.py new file mode 100644 index 000000000..b2adcaa96 --- /dev/null +++ b/lmms_eval/tasks/vitatecs/utils.py @@ -0,0 +1,225 @@ +from decord import VideoReader, cpu +import numpy as np +import os +import sys +import datetime +import lmms_eval.tasks._task_utils.file_utils as file_utils +import json +import logging +import yaml +from pathlib import Path + +import requests +import openai +from openai import OpenAI +import time +import ast +from tqdm import tqdm +import random + +import re + +with open(Path(__file__).parent / "_default_template_yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + + +API_TYPE = os.getenv("API_TYPE", "openai") + +if API_TYPE == "openai": + API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") + API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") + headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json", + } + +# We will unzip all the zip files +# To HF HOME cache dir +# And load it here +HF_HOME = os.environ["HF_HOME"] +cache_dir = config["dataset_kwargs"]["cache_dir"] +cache_dir = os.path.join(HF_HOME, cache_dir) + +eval_logger = logging.getLogger("lmms-eval") + + +# Pass in video path here +# Can only work correctly with video llm +def vitatecs_doc_to_visual(doc): + video_path = os.path.join(cache_dir, doc["src_dataset"], doc["video_name"]) + if os.path.exists(video_path): + video_path = video_path + else: + sys.exit(f"video path:{video_path} does not exist, please check") + return [video_path] + + +# This is the place where you format your question +def vitatecs_doc_to_text(doc, model_specific_prompt_kwargs=None): + if model_specific_prompt_kwargs is None: + model_specific_prompt_kwargs = {} + pre_prompt = "" + post_prompt = "" + if "pre_prompt" in model_specific_prompt_kwargs: + pre_prompt = model_specific_prompt_kwargs["pre_prompt"] + if "post_prompt" in model_specific_prompt_kwargs: + post_prompt = model_specific_prompt_kwargs["post_prompt"] + + question, _, _ = format_question_and_answer(doc) + return f"{pre_prompt}{question}{post_prompt}" + + +def process_option_for_question(sent): + if not sent.endswith("."): + sent += "." + return sent.capitalize() + + +def process_option_for_matching(sent): + if sent.endswith("."): + sent = sent[:-1] + return sent.lower() + + +def format_question_and_answer(doc): + seed = sum(ord(c) for c in doc["caption"] + doc["counterfactual"]) % 100 + random.seed(seed) + if random.random() > 0.5: + option_a = process_option_for_question(doc["caption"]) + option_b = process_option_for_question(doc["counterfactual"]) + answer = "(A) " + option_a + else: + option_a = process_option_for_question(doc["counterfactual"]) + option_b = process_option_for_question(doc["caption"]) + answer = "(B) " + option_b + options = [process_option_for_matching(doc["caption"]), process_option_for_matching(doc["counterfactual"])] + + question = f"Which of the following best describes the content of the video: \n(A) {option_a} \n(B) {option_b}" + return question, answer, options + + +def vitatecs_doc_to_answer(doc): + _, answer, _ = format_question_and_answer(doc) + return answer + + +# Process result +def vitatecs_process_results(doc, result): + pred = result[0] + rating = 0 + match_success = True + chatgpt_response = None + question, answer, options = format_question_and_answer(doc) + + # Some hand-crafted matching rules + if options[0] in pred.lower() and options[1] not in pred.lower(): + rating = 1 + elif options[1] in pred.lower() and options[0] not in pred.lower(): + rating = 0 + elif pred in ["A", "B"]: + rating = 1 if pred == answer[1] else 0 + elif any(pred.startswith(prefix) for prefix in ["A.", "B."]): + rating = 1 if pred.split(".")[0] == answer[1] else 0 + elif any(pred.startswith(prefix) for prefix in ["A)", "B)"]): + rating = 1 if pred.split(")")[0] == answer[1] else 0 + elif any(pred.startswith(prefix) for prefix in ["(A)", "(B)"]): + rating = 1 if pred.split(")")[1] == answer[1] else 0 + else: + # Fail to match answer in the video-llm response. Use ChatGPT to evaluate. + match_success = False + + base_prompt = """You will receive a caption matching question, the ground-truth answer and the prediction from a question answering (QA) model. Your task is to determine whether QA model prediction is correct, based on the question and ground-truth answer. If the prediction is correct, respond "Correct". If the prediction is incorrect, respond "Incorrect". """ + prompt = f"""{base_prompt}\n\nCaption Matching Question: {question}\n\nGround-Truth Answer: {answer}\n\nModel Prediction: {pred}""" + chatgpt_response, rating = get_eval_result(prompt) + + if not match_success: + return { + "accuracy": { + "src_dataset": doc["src_dataset"], + "video_id": doc["video_name"], + "question": question, + "gt-answer": answer, + "video-llm-prediction": pred, + "match_success": match_success, + "rating": rating, + # "chatgpt_prompt": prompt, + "chatgpt_response": chatgpt_response, + "aspect": doc["aspect"], + }, + } + else: + return { + "accuracy": { + "src_dataset": doc["src_dataset"], + "video_id": doc["video_name"], + "question": question, + "gt-answer": answer, + "video-llm-prediction": pred, + "match_success": match_success, + "rating": rating, + "aspect": doc["aspect"], + }, + } + + +# utils function for gpt_evaluation when rule-based matching is unsuccessful +def get_eval_result(prompt, maxtry=10, sys_prompt=None): + llm_output = None + while True: + try: + llm_output = get_llm_output(prompt, sys_prompt) + rating = llm_output_to_rating(llm_output) + return llm_output, rating + except: + if maxtry <= 0: + return llm_output, 0 + maxtry -= 1 + print(f"Not success! {maxtry} retries remaining...") + time.sleep(random.uniform(1, 2)) + + +# utils function for gpt evaluation +def get_llm_output(prompt, sys_prompt, max_tokens=128): + if sys_prompt is None: + sys_prompt = "You are an AI assistant for question answering." + data = {"max_tokens": max_tokens, "model": "gpt-3.5-turbo-1106", "temperature": 1.0, "top_p": 1, "presence_penalty": 1, "messages": [{"role": "system", "content": sys_prompt}, {"role": "user", "content": prompt}]} + response = requests.post(API_URL, headers=headers, data=json.dumps(data).encode("utf-8")) + result = response.content.decode("utf-8") + dict_result = json.loads(result) + llm_output = dict_result["choices"][0]["message"]["content"].strip() + return llm_output + + +# utils function that converts gpt evaluation into rating +def llm_output_to_rating(llm_output): + assert "Correct" in llm_output or "Incorrect" in llm_output + if llm_output.startswith("Correct"): + rating = 1 + elif llm_output.startswith("Incorrect"): + rating = 0 + elif ("Correct" in llm_output) and ("Incorrect" not in llm_output): + rating = 1 + elif "Incorrect" in llm_output: + rating = 0 + return rating + + +# Factory into different aggregate +def vitatecs_aggregate_rating(results, args): + yes_count = 0 + + # results is a list of dict + for answer_dict in results: + if answer_dict["rating"] == 1: + yes_count += 1 + + accuracy = yes_count / len(results) + + return accuracy * 100 diff --git a/lmms_eval/tasks/vitatecs/vitatecs_compositionality.yaml b/lmms_eval/tasks/vitatecs/vitatecs_compositionality.yaml new file mode 100644 index 000000000..eb73ef00d --- /dev/null +++ b/lmms_eval/tasks/vitatecs/vitatecs_compositionality.yaml @@ -0,0 +1,13 @@ +dataset_name: "Compositionality" +task: "vitatecs_compositionality" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.vitatecs_doc_to_visual +doc_to_text: !function utils.vitatecs_doc_to_text +doc_to_target: !function utils.vitatecs_doc_to_answer +process_results: !function utils.vitatecs_process_results +metric_list: + - metric: accuracy + aggregation: !function utils.vitatecs_aggregate_rating + higher_is_better: true +include: _default_template_yaml diff --git a/lmms_eval/tasks/vitatecs/vitatecs_direction.yaml b/lmms_eval/tasks/vitatecs/vitatecs_direction.yaml new file mode 100644 index 000000000..8c4b5b60b --- /dev/null +++ b/lmms_eval/tasks/vitatecs/vitatecs_direction.yaml @@ -0,0 +1,13 @@ +dataset_name: "Direction" +task: "vitatecs_direction" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.vitatecs_doc_to_visual +doc_to_text: !function utils.vitatecs_doc_to_text +doc_to_target: !function utils.vitatecs_doc_to_answer +process_results: !function utils.vitatecs_process_results +metric_list: + - metric: accuracy + aggregation: !function utils.vitatecs_aggregate_rating + higher_is_better: true +include: _default_template_yaml diff --git a/lmms_eval/tasks/vitatecs/vitatecs_intensity.yaml b/lmms_eval/tasks/vitatecs/vitatecs_intensity.yaml new file mode 100644 index 000000000..a12a4dea2 --- /dev/null +++ b/lmms_eval/tasks/vitatecs/vitatecs_intensity.yaml @@ -0,0 +1,13 @@ +dataset_name: "Intensity" +task: "vitatecs_intensity" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.vitatecs_doc_to_visual +doc_to_text: !function utils.vitatecs_doc_to_text +doc_to_target: !function utils.vitatecs_doc_to_answer +process_results: !function utils.vitatecs_process_results +metric_list: + - metric: accuracy + aggregation: !function utils.vitatecs_aggregate_rating + higher_is_better: true +include: _default_template_yaml diff --git a/lmms_eval/tasks/vitatecs/vitatecs_localization.yaml b/lmms_eval/tasks/vitatecs/vitatecs_localization.yaml new file mode 100644 index 000000000..633fae76e --- /dev/null +++ b/lmms_eval/tasks/vitatecs/vitatecs_localization.yaml @@ -0,0 +1,13 @@ +dataset_name: "Localization" +task: "vitatecs_localization" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.vitatecs_doc_to_visual +doc_to_text: !function utils.vitatecs_doc_to_text +doc_to_target: !function utils.vitatecs_doc_to_answer +process_results: !function utils.vitatecs_process_results +metric_list: + - metric: accuracy + aggregation: !function utils.vitatecs_aggregate_rating + higher_is_better: true +include: _default_template_yaml diff --git a/lmms_eval/tasks/vitatecs/vitatecs_sequence.yaml b/lmms_eval/tasks/vitatecs/vitatecs_sequence.yaml new file mode 100644 index 000000000..6ad434917 --- /dev/null +++ b/lmms_eval/tasks/vitatecs/vitatecs_sequence.yaml @@ -0,0 +1,13 @@ +dataset_name: "Sequence" +task: "vitatecs_sequence" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.vitatecs_doc_to_visual +doc_to_text: !function utils.vitatecs_doc_to_text +doc_to_target: !function utils.vitatecs_doc_to_answer +process_results: !function utils.vitatecs_process_results +metric_list: + - metric: accuracy + aggregation: !function utils.vitatecs_aggregate_rating + higher_is_better: true +include: _default_template_yaml diff --git a/lmms_eval/tasks/vitatecs/vitatecs_type.yaml b/lmms_eval/tasks/vitatecs/vitatecs_type.yaml new file mode 100644 index 000000000..7034a45c5 --- /dev/null +++ b/lmms_eval/tasks/vitatecs/vitatecs_type.yaml @@ -0,0 +1,13 @@ +dataset_name: "Type" +task: "vitatecs_type" +test_split: test +output_type: generate_until +doc_to_visual: !function utils.vitatecs_doc_to_visual +doc_to_text: !function utils.vitatecs_doc_to_text +doc_to_target: !function utils.vitatecs_doc_to_answer +process_results: !function utils.vitatecs_process_results +metric_list: + - metric: accuracy + aggregation: !function utils.vitatecs_aggregate_rating + higher_is_better: true +include: _default_template_yaml diff --git a/pyproject.toml b/pyproject.toml index e01140ea8..cb9f04e6a 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "lmms_eval" -version = "0.2.0.post1" +version = "0.2.1" authors = [ { name = "LMMMs-Lab Evaluation Team", email = "lmms_eval@outlook.com" }, ] @@ -74,6 +74,8 @@ dependencies = [ "spacy", "anls", "rouge", + "capture_metric", + "protobuf==3.20", ] [project.optional-dependencies] diff --git a/tools/live_bench/example.ipynb b/tools/live_bench/example.ipynb deleted file mode 100644 index 92b9df73e..000000000 --- a/tools/live_bench/example.ipynb +++ /dev/null @@ -1,481 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/pufanyi/anaconda3/anacondabin/envs/live_bench/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from random import sample\n", - "\n", - "from live_bench.websites.website import DefaultWebsite\n", - "from live_bench.websites import load_websites\n", - "\n", - "# website = load_websites()\n", - "# website = sample(website, 1)\n", - "# website[0].url\n", - "website = [DefaultWebsite(url=\"https://www.asahi.com/\")] # , DefaultWebsite(url=\"https://www.bbc.com/sport\"), DefaultWebsite(url=\"https://www.bbc.com/business\")]" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "could not detect version_main.therefore, we are assuming it is chrome 108 or higher\n" - ] - } - ], - "source": [ - "from live_bench.data_generator.utils.extract_infomation import InfomationExtractor\n", - "from live_bench.screen_shoter import get_shoter\n", - "from live_bench.driver import load_driver\n", - "\n", - "shoter = get_shoter(\"single_screen\")\n", - "driver = load_driver()\n", - "w = shoter(driver, website[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "extractor = InfomationExtractor()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "response = extractor.extract_infomation(w)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "**Here is something you can take as reference.**\n", - "\n", - "## Text Extracted in the HTML\n", - "\n", - "Below is the text extracted from the website for you to take reference:\n", - "BBC Home - Breaking News, World News, US News, Sports, Business, Innovation, Climate, Culture, Travel, Video & Audio\n", - "\n", - "What son's conviction means for President Biden\n", - "The guilty verdict is unlikely to change voters' minds, but it will be a personal blow for the US president.\n", - "1 hr ago | US & Canada\n", - "\n", - "Hunter Biden found guilty on all counts in gun trial\n", - "The US president's son is found guilty of lying about his drug use when buying a handgun in 2018.\n", - "- The struggles and scandals of Hunter Biden\n", - "\n", - "Blinken says fate of ceasefire plan down to Hamas\n", - "The US diplomat says Israel's prime minister \"reaffirmed his commitment\" to a Gaza ceasefire plan.\n", - "2 hrs ago | Middle East\n", - "\n", - "Ukraine 'hits missile launch sites in Russia'\n", - "The mayor of the city of Kharkiv says the situation there is \"calmer\" as Russia has been shelling less.\n", - "1 hr ago | Europe\n", - "\n", - "Four US college instructors stabbed in public park in China\n", - "The instructors were on a daytime visit to a public park when they were attacked, Cornell College says.\n", - "4 hrs ago | Asia\n", - "\n", - "Animal-rights protesters attack portrait of King\n", - "Animal-rights protesters attack a portrait of King Charles III, in a London gallery.\n", - "2 hrs ago | UK\n", - "\n", - "Warning shots from South as NK soldiers cross border\n", - "The incident at the DMZ comes at a time of heightened tensions between the two Koreas.\n", - "11 hrs ago | Asia\n", - "\n", - "## Image Features\n", - "\n", - "From the screenshot of the news website you provided, here is the analysis based on the images displayed alongside the corresponding news headings and text:\n", - "\n", - "1. **Image Associated with Hunter Biden's Conviction**:\n", - " - **Description**: The image depicts Hunter Biden escorted by, possibly, security personnel or aides. He seems to be dressed in a formal dark suit and appears to be descending stairs or possibly exiting a vehicle. He carries an air of seriousness, likely reflective of the gravity of his legal situation.\n", - " - **Relevance**: This image effectively captures the serious, personal, and public nature of the judicial proceedings against the President's son, making the situation more relatable to the audience. It directly ties to the news confirming Hunter Biden’s guilty verdict in a gun trial related to lying about drug use.\n", - "\n", - "2. **Image Accompanying the Article on Biden's Supporters**:\n", - " - **Description**: The accompanying image shows a group of enthusiastic supporters holding signs, with one prominently reading \"Say Yes to Biden,\" suggesting a political rally or campaign event. The participants display expressions of support and enthusiasm.\n", - " - **Relevance**: This image provides a visual contrast to the first, highlighting the ongoing support for the Biden family or campaign despite the legal issues faced by Hunter Biden. It serves to illustrate the political backdrop and public opinion dynamic mentioned in the news headlines.\n", - "\n", - "These images serve different purposes:\n", - "- The first image personalizes the news story, putting a face to the name in a high-stakes legal controversy. It underlines the personal and public challenges faced by the Biden family due to the conviction.\n", - "- The second image contextualizes the broader political support for the Biden family, suggesting that despite personal legal woes, there is a segment of the populace fervently supporting them.\n", - "\n", - "The clear connection between the images and the corresponding textual content on the news site helps readers visualize and better understand the unfolding events, enhancing the impact of the news storytelling.\n", - "\n", - "## Interesting Points\n", - "\n", - "The BBC news website, as demonstrated through the detailed examination of its content, offers a dynamic and visually engaging approach to news presentation. Here’s a deeper analysis of how it distinguishes itself:\n", - "\n", - "1. **Comprehensive and Geographically Diverse News Coverage**:\n", - " - The content spans a wide range of geographical locations including the US, Middle East, Europe, Asia, and the UK. Each news piece targets a major recent event, reflecting the website’s commitment to global news coverage. This expansive geographic focus ensures that readers have access to a broad spectrum of significant, impactful news.\n", - "\n", - "2. **Varied Content Themes**: \n", - " - The news themes are diverse, covering political, social, and cultural issues. From the legal troubles of a high-profile political figure’s son in the US to a ceasefire plan in the Middle East and violent incidents in Asia, the website covers a wide array of topics. This variety meets different readers' interests and keeps the content engaging.\n", - "\n", - "3. **Immediate Relevance**:\n", - " - The website's content is timely, as indicated by timestamps such as “1 hr ago” and “2 hrs ago.” This reflects the website’s commitment to providing the latest news, which is crucial for maintaining reader engagement and trust in a digital age where current information is highly valued.\n", - "\n", - "4. **Stylistic and Engaging Visual Design**:\n", - " - The use of compelling images alongside the news articles plays a critical role in storytelling. For instance, the image of Hunter Biden descending steps with a serious demeanor visually reinforces the gravity of the news about his conviction. \n", - " - Meanwhile, the image of supporters holding \"Say Yes to Biden\" signs juxtaposed with Hunter Biden's legal news offers a visual narrative of continued support amidst political strife, underscoring the complexity and depth of public and personal life in politics.\n", - "\n", - "5. **Interactive and Multimedia Features**:\n", - " - The use of tags such as \"OLIVE\" beside the breaking story of Hunter Biden indicates an interactive or breaking news feature that likely offers real-time updates and extensive coverage. This kind of multimedia integration enhances user interaction and engagement.\n", - "\n", - "In summary, the BBC news website sets itself apart through a combination of up-to-date, visually engaging, and comprehensively covered news items that cater to a global audience with varied interests. The effective use of images not only contextualizes the stories but also adds a layer of emotional and visual impact, making the news relatable and striking.\n" - ] - } - ], - "source": [ - "print(str(response))" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This screenshot from a news website contains several images corresponding to different news stories. Let's examine each image and extract relevant details:\n", - "\n", - "1. **Image associated with the Michael Mosley news story:**\n", - " - The image depicts a middle-aged couple, smiling warmly at each other in a sunny, natural outdoor setting. This photo likely portrays Dr. Michael Mosley with his wife, representing a human interest angle to the story about Dr. Mosley's disappearance on the Greek island of Symi. The caption, \"'We will not lose hope,' says Michael Mosley's wife,\" implies a context of hope and determination amidst difficult circumstances.\n", - "\n", - "2. **Image linked to the news about the hostages freed in Gaza:**\n", - " - This image features a group of soldiers with one individual in civilian clothing at the center, being lifted or celebrated, possibly right after a rescue scenario. The setting appears to be a rugged outdoor area, suggestive of a conflict or military zone, which aligns with the news story about hostages being freed in Gaza. The inclusion of armed personnel and a jubilant expression on the civilian's face highlights the relief and successful outcome of a dangerous operation.\n", - "\n", - "3. **Image for the Nova festival hostages news:**\n", - " - This image depicts a motorboat on clear water under bright skies, possibly implying the geographic setting related to Michael Mosley’s disappearance near the Greek island of Symi. The serene environment contrasts starkly with the concerning news of his disappearance during what might have been a routine outing or travel.\n", - "\n", - "These images serve as visual supplements to the written content, providing readers with a clearer, more immediate understanding of the stories. They help bridge the emotional and contextual gaps that pure text might leave, allowing readers to engage more deeply with the news events. Each image is carefully selected to evoke specific sentiments and to provide visual context to the news headlines and summaries.\n" - ] - } - ], - "source": [ - "print(response[\"features\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from live_bench import LiveBench" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = LiveBench(force_clear=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2024-06'" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.name" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "could not detect version_main.therefore, we are assuming it is chrome 108 or higher\n", - "Capturing websites: 0%| | 0/1 [00:00 str: - subtask = subtask.strip().lower() - for valid_subtask in SUBTASKS: - if valid_subtask.lower() in subtask.lower(): - return valid_subtask - return "Unknown Subtask" - - def set_subtask(self, subtask: str): - """ - Set the subtask for the QAData instance after parsing it. - - Args: - subtask (str): The subtask string to be set. - """ - self.subtask = self.parse_subtask(subtask) - - def to_dict(self): - return {"question": self.question, "answer": self.answer} - - -class QAGenerator(ABC): - def __init__(self, prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md")): - self.prompt_file = prompt_file - self.prompt = self._load_prompt() - - def _load_prompt(self): - with open(self.prompt_file, "r") as f: - return f.read() - - def __call__(self, images: ScreenImage, *args, **kwargs): - return self.generate(images, *args, **kwargs) - - def generate(self, images: ScreenImage, *, test=False, infomation=None, **kwargs) -> Response: - if test: - return Response(success=True, content="This is a test response.", full_log={}) - return self._generate(images, infomation=infomation, test=test, **kwargs) - - def check(self, images: ScreenImage, question, answer, criteria, subtask, *, infomation=None, test=False, **kwargs) -> Response: - if test: - return Response(success=True, content="This is a test response.", full_log={}) - return self._check(images, question, answer, criteria, subtask, infomation=infomation, **kwargs) - - @abstractmethod - def _generate(self, images: ScreenImage, **kwargs) -> Response: - raise NotImplementedError("_generate not implemented") - - @abstractmethod - def _check(self, images: ScreenImage, question, answer, criteria, subtask, **kwargs) -> Response: - raise NotImplementedError("_check not implemented") - - def format_response(self, response: Response) -> QAData: - if response.success: - qa_data = self._format_response(response) - if qa_data is None: - return [] - else: - return qa_data - else: - return [] - - @abstractmethod - def _format_response(self, response: Response) -> str: - raise NotImplementedError("format_response not implemented") - - @abstractmethod - def format_checked_response(self, response: Response) -> QAData: - raise NotImplementedError("format_checked_response not implemented") - - def get_name(self) -> str: - raise NotImplementedError("get_name not implemented") - - -class GeneratorRegistry: - def __init__(self): - self.generators = {} - - def register_generator(self, name): - def decorator(cls): - self.generators[name] = cls - cls.get_name = lambda self: name - return cls - - return decorator - - def get_generator(self, name) -> QAGenerator: - return self.generators[name] - - def get_random_generator(self) -> QAGenerator: - return random.choice(list(self.generators.values())) - - -generator_registry = GeneratorRegistry() - - -def register_generator(name): - return generator_registry.register_generator(name) - - -def get_generator(name, *args, **kwargs) -> QAGenerator: - return generator_registry.get_generator(name)(*args, **kwargs) - - -def get_random_generator(*args, **kwargs) -> QAGenerator: - return generator_registry.get_random_generator()(*args, **kwargs) - - -@register_generator("gpt4v") -class GPT4Generator(QAGenerator): - def __init__( - self, - prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), - model="gpt-4o", - example_path=os.path.join(os.path.dirname(__file__), "example"), - check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), - ): - super().__init__(prompt_file) - API_KEY = os.getenv("OPENAI_API_KEY") - if not API_KEY: - raise ValueError("OPENAI_API_KEY environment variable not set.") - self.api_key = API_KEY - self.client = openai.OpenAI(api_key=self.api_key) - self.model = model - if os.path.exists(example_path): - self.example_path = example_path - else: - self.example_path = None - if os.path.exists(check_prompt): - with open(check_prompt, "r") as f: - self.check_prompt = f.read() - else: - self.check_prompt = check_prompt - - def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation): - example = [ - { - "type": "text", - "text": "Here are few examples about the task and the expected output format. You can take these as examples to generate your own questions.", - }, - format_gpt4v_images(example_image), - { - "type": "text", - "text": example_output, - }, - ] - content = example + [format_gpt4v_images(image) for image in images] - if infomation: - content.append({"type": "text", "text": str(infomation)}) - content.append( - { - "type": "text", - "text": "Please generate high-quality questions focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", - }, - ) - messages = [ - { - "role": "system", - "content": self.prompt, - }, - { - "role": "user", - "content": content, - }, - ] - return messages - - def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation=None, **kwargs): - if self.example_path: - example_image_path = os.path.join(self.example_path, "example_website.png") - example_output_path = os.path.join(self.example_path, "example_output.json") - example_image = Image.open(example_image_path) - with open(example_output_path, "r") as f: - example_output = json.load(f) - example_output = json.dumps(example_output, indent=4) - - messages = self.format_messages(images.images, example_image, example_output, infomation) - - return gpt4v_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) - - def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): - messages = [ - { - "role": "system", - "content": self.check_prompt, - } - ] - content = [] - for img in images: - content.append(format_gpt4v_images(img)) - content.append( - { - "type": "text", - "text": f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}\nSubtask: {subtask}", - }, - ) - if infomation: - content.append( - { - "type": "text", - "text": str(infomation), - }, - ) - content.append( - { - "type": "text", - "text": "Please rephrase or rewrite the high-quality question focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", - }, - ) - messages.append( - { - "role": "user", - "content": content, - } - ) - return messages - - def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, **kwargs): - messages = self.get_check_prompt(question, answer, criteria, subtask, images.images) - return gpt4v_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) - - def format_checked_response(self, response: Response): - data = json.loads(response.content) - question = data.get("question", None) - answer = data.get("answer", None) - criteria = data.get("criteria", None) - subtask = data.get("subtask", None) - return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) - - def _format_response(self, response: Response) -> List[QAData]: - try: - qa_data = [] - content = json.loads(response.content) - for subtask, messages in content.items(): - subtask = subtask.lower() - for message in messages: - message_lower = {k.lower(): v for k, v in message.items()} - try: - question = message_lower["question"] - answer = message_lower["answer"] - criteria = message_lower["criteria"] - qa_data.append(QAData(question=question, answer=answer, criteria=criteria, subtask=subtask)) - except KeyError as e: - logger.error(f"Failed to parse response: {message}") - logger.error(f"Error: {e}") - return qa_data - except Exception as e: - logger.error(f"Failed to format response: {e}") - return [] - - -@register_generator("gemini") -class GeminiGenerator(QAGenerator): - def __init__( - self, - prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), - model="gemini-1.5-pro-latest", - example_path=os.path.join(os.path.dirname(__file__), "example"), - check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), - ): - super().__init__(prompt_file) - GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") - if not GOOGLE_API_KEY: - raise ValueError("GOOGLE_API_KEY environment variable not set.") - genai.configure(api_key=GOOGLE_API_KEY) - - self.api_key = GOOGLE_API_KEY - self.model = model - self.client = genai.GenerativeModel(model) - if os.path.exists(example_path): - self.example_path = example_path - else: - self.example_path = None - if os.path.exists(check_prompt): - with open(check_prompt, "r") as f: - self.check_prompt = f.read() - else: - self.check_prompt = check_prompt - - def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation = None): - content = [self.prompt, "\n", "Example Image:", example_image, "\n", "Example Output:", example_output] - content.extend(images) - content.append(str(infomation)) - content.append("Please generate high-quality questions focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.") - return content - - def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation: ImageInfomation = None, **kwargs): - if self.example_path: - example_image_path = os.path.join(self.example_path, "example_website.png") - example_output_path = os.path.join(self.example_path, "example_output.json") - example_image = Image.open(example_image_path) - with open(example_output_path, "r") as f: - # example_output = f.read() - example_output = json.load(f) - example_output = json.dumps(example_output, indent=4) - - messages = self.format_messages(images.images, example_image, example_output, infomation) - - return gemini_generate_response(self.client, messages, max_tokens, max_try_times, **kwargs) - - def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): - content = [self.check_prompt] + images - content.append(f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}, Subtask: {subtask}") - content.append("Your response should be strictly in the below format:\n\nQuestion: \nAnswer: \nCriteria: \nSubtask: ") - if infomation: - content.append(str(infomation)) - return content - - def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, infomation: ImageInfomation = None, **kwargs): - messages = self.get_check_prompt(question, answer, criteria, subtask, images.images, infomation) - return gemini_generate_response(self.client, messages, max_tokens, max_try_times, **kwargs) - - def format_checked_response(self, response: Response): - # Extract the question, answer, and subtask from the normalized content - question_match = re.search(r"question:\s*(.*?)\nAnswer:", response.content, re.IGNORECASE | re.DOTALL) - answer_match = re.search(r"answer:\s*(.*?)\nCriteria", response.content, re.IGNORECASE | re.DOTALL) - criteria_match = re.search(r"criteria:\s*(.*?)\n(Subtask:|$)", response.content, re.IGNORECASE | re.DOTALL) - subtask_match = re.search(r"subtask:\s*(.*)", response.content, re.IGNORECASE) - - question = answer = subtask = None - - if question_match: - # Extract the matched groups - question = question_match.group(1).strip() - if answer_match: - answer = answer_match.group(1).strip() - if criteria_match: - criteria = criteria_match.group(1).strip() - if subtask_match: - subtask = subtask_match.group(1).strip() - - return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) - - def _format_response(self, response: Response) -> List[QAData]: - try: - qa_data = [] - content = json.loads(response.content) - for subtask, message in content.items(): - subtask = subtask.lower() - message_lower = {k.lower(): v for k, v in message.items()} - try: - question = message_lower["question"] - answer = message_lower["answer"] - qa_data.append(QAData(question=question, answer=answer, subtask=subtask)) - except KeyError as e: - logger.error(f"Failed to parse response: {message}") - logger.error(f"Error: {e}") - return qa_data - except Exception as e: - logger.error(f"Failed to format response: {e}") - return [] - - -@register_generator("claude") -class ClaudeGenerator(QAGenerator): - def __init__( - self, - prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), - model="claude-3-5-sonnet-20240620", - example_path=os.path.join(os.path.dirname(__file__), "example"), - check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), - ): - super().__init__(prompt_file) - API_KEY = os.getenv("ANTHROPIC_API_KEY") - if not API_KEY: - raise ValueError("ANTHROPIC_API_KEY environment variable not set.") - self.api_key = API_KEY - self.client = anthropic.Anthropic(api_key=self.api_key) - self.model = model - if os.path.exists(example_path): - self.example_path = example_path - else: - self.example_path = None - if os.path.exists(check_prompt): - with open(check_prompt, "r") as f: - self.check_prompt = f.read() - else: - self.check_prompt = check_prompt - - def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation): - example = [ - { - "type": "text", - "text": "Here are few examples about the task and the expected output format. You can take these as examples to generate your own questions.", - }, - format_claude_images(example_image), - { - "type": "text", - "text": example_output, - }, - ] - content = example + [format_claude_images(image) for image in images] - if infomation: - content.append({"type": "text", "text": str(infomation)}) - content.append( - { - "type": "text", - "text": "Please generate high-quality questions focusing on the information displayed within this webpage. Ensure your response adheres to the examples provided above and is structured in JSON format, incorporating regular expressions to validate the format.", - }, - ) - messages = [ - { - "role": "user", - "content": content, - }, - ] - return messages - - def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation=None, **kwargs): - if self.example_path: - example_image_path = os.path.join(self.example_path, "example_website.png") - example_output_path = os.path.join(self.example_path, "example_output.json") - example_image = Image.open(example_image_path) - with open(example_output_path, "r") as f: - # example_output = f.read() - example_output = json.load(f) - example_output = json.dumps(example_output, indent=4) - - messages = self.format_messages(images.images, example_image, example_output, infomation) - - return claude_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, system=self.prompt, **kwargs) - - def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): - messages = [ - { - "role": "system", - "content": self.check_prompt, - } - ] - content = [] - for img in images: - content.append(format_claude_images(img)) - content.append( - { - "type": "text", - "text": f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}\nSubtask: {subtask}", - }, - ) - if infomation: - content.append( - { - "type": "text", - "text": str(infomation), - }, - ) - content.append( - { - "type": "text", - "text": "Please rephrase or rewrite the high-quality question focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", - }, - ) - messages.append( - { - "role": "user", - "content": content, - } - ) - return messages - - def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, **kwargs): - messages = self.get_check_prompt(question, answer, criteria, subtask, images.images) - return claude_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) - - def format_checked_response(self, response: Response): - data = json.loads(response.content) - question = data.get("question", None) - answer = data.get("answer", None) - criteria = data.get("criteria", None) - subtask = data.get("subtask", None) - return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) - - def _format_response(self, response: Response) -> List[QAData]: - try: - qa_data = [] - content = json.loads(response.content) - for subtask, messages in content.items(): - subtask = subtask.lower() - for message in messages: - message_lower = {k.lower(): v for k, v in message.items()} - try: - question = message_lower["question"] - answer = message_lower["answer"] - criteria = message_lower["criteria"] - qa_data.append(QAData(question=question, answer=answer, criteria=criteria, subtask=subtask)) - except KeyError as e: - logger.error(f"Failed to parse response: {message}") - logger.error(f"Error: {e}") - return qa_data - except Exception as e: - logger.error(f"Failed to format response: {e}") - return [] diff --git a/tools/live_bench/live_bench/data_generator/response.py b/tools/live_bench/live_bench/data_generator/response.py deleted file mode 100644 index 9eed882be..000000000 --- a/tools/live_bench/live_bench/data_generator/response.py +++ /dev/null @@ -1,12 +0,0 @@ -class Response(object): - def __init__(self, success: bool, content: str, full_log: dict): - self.success = success - self.content = content - self.full_log = full_log - - def to_dict(self): - return { - "success": self.success, - "content": self.content, - "full_log": self.full_log, - } diff --git a/tools/live_bench/live_bench/data_generator/score_getter.py b/tools/live_bench/live_bench/data_generator/score_getter.py deleted file mode 100644 index 412989285..000000000 --- a/tools/live_bench/live_bench/data_generator/score_getter.py +++ /dev/null @@ -1,157 +0,0 @@ -import os -import json -import random -import openai -import anthropic -from abc import ABC, abstractmethod -from typing import List -from PIL import Image -from live_bench.screen_shoter import ScreenImage -from live_bench.data_generator.qa_generator import Response -from live_bench.data_generator.utils.gpt4v import format_gpt4v_images, gpt4v_generate_response -from live_bench.data_generator.utils.claude import format_claude_images, claude_generate_response - - -class Score(object): - def __init__(self, score: int, reason: str): - self.score = score - self.reason = reason - - -class ScoreGetter(ABC): - def get_name(self): - return self.name - - @abstractmethod - def get_score(self, question: str, answer: str, images: ScreenImage): - raise NotImplementedError("get_score not implemented") - - def __call__(self, question: str, answer: str, images: ScreenImage, **kwargs): - return self.get_score(question, answer, images, **kwargs) - - -class ScoreGetterRegistry: - def __init__(self): - self.score_getters = {} - - def register_score_getter(self, name): - def decorator(cls): - self.score_getters[name] = cls - cls.name = name - return cls - - return decorator - - def get_score_getter(self, name) -> ScoreGetter: - return self.score_getters[name] - - def get_random_score_getter(self) -> ScoreGetter: - return random.choice(list(self.score_getters.values())) - - -generator_registry = ScoreGetterRegistry() - - -def register_score_getter(name): - return generator_registry.register_score_getter(name) - - -def get_score_getter(name, *args, **kwargs) -> ScoreGetter: - return generator_registry.get_score_getter(name)(*args, **kwargs) - - -def get_random_score_getter(*args, **kwargs) -> ScoreGetter: - return generator_registry.get_random_score_getter()(*args, **kwargs) - - -@register_score_getter("gpt4v") -class GPT4VScoreGetter(ScoreGetter): - def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="gpt-4o", example_path=os.path.join(os.path.dirname(__file__), "example")): - super().__init__() - if os.path.exists(prompt): - with open(prompt, "r") as f: - self.prompt = f.read() - else: - self.prompt = prompt - API_KEY = os.getenv("OPENAI_API_KEY") - if not API_KEY: - raise ValueError("OPENAI_API_KEY environment variable not set.") - self.api_key = API_KEY - self.client = openai.OpenAI(api_key=self.api_key) - self.model = model - if os.path.exists(example_path) and os.path.isfile(os.path.join(example_path, "example_score_input.md")): - with open(example_path, "r") as f: - self.example = f.read() - else: - self.example = None - - def _format_prompt(self, question: str, answer: str, images: List[Image.Image]): - prompt = [{"role": "system", "content": self.prompt}] - messages = [] - for image in images: - messages.append(format_gpt4v_images(image)) - messages.append({"type": "text", "text": f"Question: {question}\nQuestioner's Answer: {answer}"}) - messages.append({"type": "text", "text": 'You should format you answer into json format like this: {"reason": "some reason", "score": 10}'}) - prompt.append({"role": "user", "content": messages}) - return prompt - - def get_score(self, question: str, answer: str, images: ScreenImage, *, max_tokens=4096, max_try_times=5, **kwargs) -> Score: - prompt = self._format_prompt(question, answer, images) - try: - response = gpt4v_generate_response(client=self.client, model=self.model, messages=prompt, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) - if response.success: - content = json.loads(response.content) - score = content.get("score", None) - reason = content.get("reason", None) - return Score(score=score, reason=reason) - else: - return Score(score=None, reason=response.content) - except Exception as e: - return Score(score=None, reason=str(e)) - - -@register_score_getter("claude") -class ClaudeScoreGetter(ScoreGetter): - def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="claude-3-5-sonnet-20240620", example_path=os.path.join(os.path.dirname(__file__), "example")): - super().__init__() - if os.path.exists(prompt): - with open(prompt, "r") as f: - self.prompt = f.read() - else: - self.prompt = prompt - API_KEY = os.getenv("ANTHROPIC_API_KEY") - if not API_KEY: - raise ValueError("ANTHROPIC_API_KEY environment variable not set.") - self.api_key = API_KEY - self.client = anthropic.Anthropic(api_key=self.api_key) - self.model = model - if os.path.exists(example_path) and os.path.isfile(os.path.join(example_path, "example_score_input.md")): - with open(example_path, "r") as f: - self.example = f.read() - else: - self.example = None - - def _format_prompt(self, question: str, answer: str, images: List[Image.Image]): - # prompt = [{"role": "system", "content": self.prompt}] - prompt = [] - messages = [] - for image in images: - messages.append(format_claude_images(image)) - messages.append({"type": "text", "text": f"Question: {question}\nQuestioner's Answer: {answer}"}) - messages.append({"type": "text", "text": 'You should format you answer into JSON format like this: { "reason": "some reason", "score": 10 }'}) - prompt.append({"role": "user", "content": messages}) - return prompt - - def get_score(self, question: str, answer: str, images: ScreenImage, *, max_tokens=4096, max_try_times=5, **kwargs) -> Score: - prompt = self._format_prompt(question, answer, images) - try: - response = claude_generate_response(client=self.client, model=self.model, messages=prompt, system=self.prompt, max_tokens=max_tokens, max_try_times=max_try_times, **kwargs) - if response.success: - content = json.loads(response.content) - score = content.get("score", None) - reason = content.get("reason", None) - return Score(score=score, reason=reason) - else: - return Score(score=None, reason=response.content) - except Exception as e: - return Score(score=None, reason=str(e)) diff --git a/tools/live_bench/live_bench/data_generator/utils/claude.py b/tools/live_bench/live_bench/data_generator/utils/claude.py deleted file mode 100644 index ccd0b381f..000000000 --- a/tools/live_bench/live_bench/data_generator/utils/claude.py +++ /dev/null @@ -1,68 +0,0 @@ -from PIL import Image -import io -import base64 -from live_bench.data_generator.response import Response -import anthropic -import logging -from time import sleep -from typing import Union, List - -logger = logging.getLogger("lmms-eval") - - -def format_claude_images(image: Union[Image.Image, List[Image.Image]]): - if isinstance(image, list): - return [format_claude_images(img) for img in image] - buffered = io.BytesIO() - image.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": img_str, - }, - } - - -def claude_generate_response(client: anthropic.Anthropic, model, messages, max_tokens: int = 4096, max_try_times=5, system=None, json_format="auto", test=False, **kwargs): - if json_format == "auto": - json_format = False - for message in messages: - if message.get("role") == "user": - contents = message.get("content", []) - if isinstance(contents, str): - if "json" in contents: - json_format = True - break - else: - for content in contents: - if content.get("type", None) == "text" and "json" in content.get("text", ""): - json_format = True - break - - if json_format: - messages.append({"role": "assistant", "content": "{"}) - - def _generate(): - if system: - return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, system=system, **kwargs) - else: - return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, **kwargs) - - for times in range(max_try_times): - try: - response = _generate() - response_str = response.content[0].text - if json_format: - response_str = "{" + response_str - return Response(success=True, content=response_str, full_log={"input": messages, "output": response.to_dict()}) - except Exception as e: - logger.error(f"Failed to generate response: {e}") - if times < max_try_times - 1: - logger.info(f"Retrying... ({times+1}/{max_try_times})") - sleep(3) - else: - logger.error("Failed to generate response after retrying.") - return Response(success=False, content=str(e), full_log={"input": messages, "output": None}) diff --git a/tools/live_bench/live_bench/data_generator/utils/gemini.py b/tools/live_bench/live_bench/data_generator/utils/gemini.py deleted file mode 100644 index 57b77536c..000000000 --- a/tools/live_bench/live_bench/data_generator/utils/gemini.py +++ /dev/null @@ -1,37 +0,0 @@ -import google.generativeai as genai -from time import sleep -from live_bench.data_generator.response import Response -import logging -from google.generativeai.types import HarmCategory, HarmBlockThreshold - -logger = logging.getLogger("lmms-eval") - - -def gemini_generate_response(client: genai.GenerativeModel, messages, max_tokens: int, max_try_times: int = 5, **kwargs): - generation_config = genai.GenerationConfig(max_output_tokens=max_tokens) - - def _generate(): - return client.generate_content( - messages, - generation_config=generation_config, - safety_settings={ - HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, - }, - **kwargs, - ) - - for times in range(max_try_times): - try: - response = _generate() - return Response(success=True, content=response.text, full_log={"input": messages, "output": response}) - except Exception as e: - logger.error(f"Failed to generate response: {e}") - if times < max_try_times - 1: - logger.info(f"Retrying... ({times+1}/{max_try_times})") - sleep(3) - else: - logger.error("Failed to generate response after retrying.") - return Response(success=False, content=str(e), full_log={"input": messages, "output": None}) diff --git a/tools/live_bench/live_bench/driver/.gitignore b/tools/live_bench/live_bench/driver/.gitignore deleted file mode 100644 index 0ef18421a..000000000 --- a/tools/live_bench/live_bench/driver/.gitignore +++ /dev/null @@ -1 +0,0 @@ -extensions/ diff --git a/tools/live_bench/live_bench/driver/load_driver.py b/tools/live_bench/live_bench/driver/load_driver.py deleted file mode 100644 index c0ca7528d..000000000 --- a/tools/live_bench/live_bench/driver/load_driver.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -import zipfile -import requests -from selenium import webdriver -from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.core.os_manager import ChromeType -from selenium.webdriver.chrome.options import Options - -import undetected_chromedriver as uc - - -def load_driver( - window_size="auto", - headless=True, - driver="undetected_chromedriver", - driver_version=None, - chrome_type="CHROME", - adblock=True, - adblock_version="6.0.2-mv3", - extension_cache_dir=os.path.join(os.path.dirname(__file__), "extensions"), - *, - service=None, - additional_options=None, -): - options = Options() - if service is None: - chrome_type = chrome_type.upper() - if chrome_type == "CHROMIUM": - chrome_type = ChromeType.CHROMIUM - elif chrome_type == "CHROME": - chrome_type = ChromeType.GOOGLE - elif chrome_type == "BRAVE": - chrome_type = ChromeType.BRAVE - service = ChromeDriverManager(driver_version=driver_version, chrome_type=chrome_type).install() - if headless: - options.add_argument("--headless") - if adblock: - try: - adblock_url = f"https://code.getadblock.com/releases/adblockchrome-{adblock_version}.zip" - adblock_path = os.path.join(extension_cache_dir, f"adblockchrome-{adblock_version}") - if not os.path.isdir(adblock_path): - os.makedirs(os.path.join(adblock_path, ".."), exist_ok=True) - # Download the adblock zip file - response = requests.get(adblock_url) - with open(f"{adblock_path}.zip", "wb") as file: - file.write(response.content) - # Unzip the downloaded file - with zipfile.ZipFile(f"{adblock_path}.zip", "r") as zip_ref: - zip_ref.extractall(adblock_path) - # Remove the zip file after extraction - os.remove(f"{adblock_path}.zip") - options.add_argument(f"--load-extension={os.path.abspath(adblock_path)}") - except Exception as e: - print(f"Error loading adblock extension: {e}") - if driver == "undetected_chromedriver": - driver = uc.Chrome(headless=headless, options=options, driver_executable_path=service) - if window_size != "auto": - driver.set_window_size(*window_size) - return driver - elif driver == "chrome": - options = Options() - if additional_options is not None: - for option in additional_options: - options.add_argument(option) - service = webdriver.chrome.service.Service(service) - driver = webdriver.Chrome(service=service, options=options) - if window_size != "auto": - driver.set_window_size(*window_size) - return driver - else: - raise ValueError(f"Unknown driver: {driver}") diff --git a/tools/live_bench/live_bench/screen_shoter/screen.py b/tools/live_bench/live_bench/screen_shoter/screen.py deleted file mode 100644 index 5817b8cc6..000000000 --- a/tools/live_bench/live_bench/screen_shoter/screen.py +++ /dev/null @@ -1,30 +0,0 @@ -import io -import base64 - -from PIL import Image -from typing import List, Tuple - -from live_bench.websites import Website - - -def image_to_base64(image: Image.Image) -> str: - buffered = io.BytesIO() - image.save(buffered, format="PNG") - return base64.b64encode(buffered.getvalue()).decode("utf-8") - - -class ScreenImage(object): - def __init__(self, images: List[Image.Image], website: Website, shoter: str, screen_size: Tuple[int, int], capture_datetime: str): - self.images = images - self.website = website - self.shoter = shoter - self.screen_size = screen_size - self.capture_datetime = capture_datetime - - def to_dict(self): - return {"images": self.images, "website": self.website.get_info(), "shoter": self.shoter, "screen_size": self.screen_size, "capture_datetime": self.capture_datetime} - - def to_output_dict(self): - output = self.to_dict() - output["images"] = [image_to_base64(image) for image in self.images] - return output diff --git a/tools/live_bench/live_bench/screen_shoter/screen_shoter.py b/tools/live_bench/live_bench/screen_shoter/screen_shoter.py deleted file mode 100644 index 6069cd26d..000000000 --- a/tools/live_bench/live_bench/screen_shoter/screen_shoter.py +++ /dev/null @@ -1,141 +0,0 @@ -from selenium import webdriver -from PIL import Image -from live_bench.websites import Website -from live_bench.screen_shoter.screen import ScreenImage -from typing import List -from abc import ABC, abstractmethod -from datetime import datetime -from PIL import Image -import os -import io -import logging - -logger = logging.getLogger("lmms-eval") - - -class ScreenShoter(ABC): - def __init__(self, screen_size=(1024, 1024)): - self.screen_size = screen_size - - def capture(self, driver: webdriver.Chrome, website: Website) -> ScreenImage: - if driver is not None: - website.visit(driver) - if self.screen_size != "auto": - driver.set_window_size(self.screen_size[0], self.screen_size[1]) - else: - driver.set_window_size(1024, 1024) - page_width = driver.execute_script("return document.body.scrollWidth") - driver.set_window_size(page_width, 1024) - # print("Screen size:", driver.get_window_size()) - images = self.get_screenshot(driver) - return ScreenImage(images, website, self.get_name(), self.screen_size, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - - def __call__(self, driver: webdriver.Chrome, website: Website) -> List[Image.Image]: - return self.capture(driver, website) - - def get_name(self) -> str: - raise NotImplementedError("get_name not implemented") - - @abstractmethod - def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: - pass - - -class ScreenShoterRegistry: - def __init__(self): - self.shoters = {} - - def register_shoter(self, name): - def decorator(cls): - self.shoters[name] = cls - cls.get_name = lambda self: name - return cls - - return decorator - - def get_shoter(self, name) -> ScreenShoter: - return self.shoters[name] - - -shoter_registry = ScreenShoterRegistry() - - -def register_shoter(name): - return shoter_registry.register_shoter(name) - - -def get_shoter(name, *args, **kwargs) -> ScreenShoter: - return shoter_registry.get_shoter(name)(*args, **kwargs) - - -@register_shoter("human") -class HumanScreenShoter(ScreenShoter): - def __init__(self, screen_size=None): - super().__init__(screen_size) - - def capture(self, driver: webdriver.Chrome, website: Website) -> ScreenImage: - path = website.get_path() - images = [] - - def get_image(path): - try: - with open(path, "rb") as f: - image_data = f.read() - image = Image.open(io.BytesIO(image_data)) - images.append(image) - except Exception as e: - logger.error(f"Error loading image {path}: {e}") - - if os.path.isdir(path): - for root, dirs, files in os.walk(path): - for file_name in files: - get_image(os.path.join(root, file_name)) - else: - try: - get_image(path) - except Exception as e: - logger.error(f"Error loading image {path}: {e}") - if not images: - raise ValueError(f"No images found in {path}") - return ScreenImage(images, website, self.get_name(), self.screen_size, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - - def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: - return [] - - -@register_shoter("single_screen") -class SingleScreenShoter(ScreenShoter): - def __init__(self, screen_size=(1024, 1024)): - super().__init__(screen_size) - - def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: - screenshot = driver.get_screenshot_as_png() - return [Image.open(io.BytesIO(screenshot))] - - -@register_shoter("rolling_screen") -class RollingScreenShoter(ScreenShoter): - def __init__(self, screen_size=(1024, 1024)): - super().__init__(screen_size) - - def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: - screenshots = [] - # Scroll to the top of the page before taking the first screenshot - driver.execute_script("window.scrollTo(0, 0)") - # Get the total height of the web page - total_height = driver.execute_script("return document.body.parentNode.scrollHeight") - # Get the viewport height - viewport_height = driver.execute_script("return window.innerHeight") - # Initialize the current scroll position - current_scroll_position = 0 - - # Scroll through the page and take screenshots - while current_scroll_position < total_height: - # Take screenshot and append to the list - screenshot = driver.get_screenshot_as_png() - screenshots.append(Image.open(io.BytesIO(screenshot))) - # Scroll down by the viewport height - current_scroll_position += viewport_height - driver.execute_script(f"window.scrollTo(0, {current_scroll_position})") - - return screenshots diff --git a/tools/live_bench/live_bench/websites/load_website.py b/tools/live_bench/live_bench/websites/load_website.py deleted file mode 100644 index 10976db75..000000000 --- a/tools/live_bench/live_bench/websites/load_website.py +++ /dev/null @@ -1,34 +0,0 @@ -import yaml -import os -from random import sample -from live_bench.websites.website import Website, DefaultWebsite, HumanScreenShotWebsite - - -def get_website(website_dict): - if "website_class" not in website_dict: - website_class = DefaultWebsite - else: - website_class = website_dict["website_class"] - url = website_dict["url"] - if "args" in website_dict: - return website_class(url, **website_dict["args"]) - else: - return website_class(url) - - -def load_websites(num_sample: int = -1): - website_list_path = os.path.join(os.path.dirname(__file__), "website_list.yaml") - with open(website_list_path, "r") as f: - website_list = yaml.full_load(f)["websites"] - if num_sample > 0: - website_list = sample(website_list, num_sample) - return [get_website(website_dict) for website_dict in website_list] - - -def load_websites_from_file(file_path): - names = os.listdir(file_path) - websites = [] - for name in names: - path = os.path.join(file_path, name) - websites.append(HumanScreenShotWebsite(path=path, name=name)) - return websites diff --git a/tools/live_bench/live_bench/websites/website.py b/tools/live_bench/live_bench/websites/website.py deleted file mode 100644 index 327bad372..000000000 --- a/tools/live_bench/live_bench/websites/website.py +++ /dev/null @@ -1,62 +0,0 @@ -import time -import os - -from webdriver_manager.core.driver import Driver -from abc import ABC, abstractmethod - - -class Website(ABC): - def __init__(self, url=None, name=None, path=None): - self.url = url - self.name = name - self.path = path - assert self.url is not None or self.path is not None, "Either url or path must be provided" - - def get_path(self): - if self.url: - return self.url - else: - return self.path - - def visit(self, driver: Driver): - self.pre_visit(driver) - driver.get(self.url) - self.post_visit(driver) - - def get_info(self): - info = {} - if self.url: - info["url"] = self.url - if self.name: - info["name"] = self.name - return info - - @abstractmethod - def pre_visit(self, driver: Driver): - raise NotImplementedError("pre_action not implemented") - - @abstractmethod - def post_visit(self, driver: Driver): - raise NotImplementedError("post_action not implemented") - - -class DefaultWebsite(Website): - def __init__(self, url, name=None): - super().__init__(url, name) - - def pre_visit(self, driver: Driver): - pass - - def post_visit(self, driver: Driver): - time.sleep(5) # Wait for 5 seconds to allow adblock to finish - - -class HumanScreenShotWebsite(Website): - def __init__(self, name=None, path=None): - super().__init__(name=name, path=path) - - def pre_visit(self, driver: Driver): - pass - - def post_visit(self, driver: Driver): - pass diff --git a/tools/live_bench/live_bench/websites/website_list.yaml b/tools/live_bench/live_bench/websites/website_list.yaml deleted file mode 100644 index c85605d7d..000000000 --- a/tools/live_bench/live_bench/websites/website_list.yaml +++ /dev/null @@ -1,78 +0,0 @@ -websites: -- url: https://www.bbc.com/ - # can add below line to specify the class to use for this website - # website_class: !constructor website.DefaultWebsite - # can add args tag to specify the arguments to pass to the class constructor - # args: - # arg1: value1 - # arg2: value2 -# - url: https://www.bbc.com/news -# - url: https://www.bbc.com/sport -# - url: https://www.bbc.com/business -# - url: https://www.bbc.com/innovation -# - url: https://www.bbc.com/culture -# - url: https://www.bbc.com/travel -# - url: https://www.bbc.com/future-planet -# - url: https://edition.cnn.com/ -# - url: https://edition.cnn.com/politics -# - url: https://edition.cnn.com/entertainment -# - url: https://edition.cnn.com/style -# - url: https://www.bloomberg.com/economics -# - url: https://www.bloomberg.com/industries -# - url: https://www.bloomberg.com/technology -# - url: https://www.bloomberg.com/politics -# - url: https://www.bloomberg.com/opinion -# - url: https://www.wsj.com/ -# - url: https://www.wsj.com/world/africa?mod=nav_top_subsection -# - url: https://www.wsj.com/world/americas?mod=nav_top_subsection -# - url: https://www.wsj.com/world/asia?mod=nav_top_subsection -# - url: https://www.wsj.com/world/china?mod=nav_top_subsection -# - url: https://www.wsj.com/world/europe?mod=nav_top_subsection -# - url: https://www.wsj.com/world/middle-east?mod=nav_top_subsection -# - url: https://www.wsj.com/world/india?mod=nav_top_subsection -# - url: https://www.wsj.com/world/oceania?mod=nav_top_subsection -# - url: https://www.wsj.com/world/russia?mod=nav_top_subsection -# - url: https://www.wsj.com/world/uk?mod=nav_top_subsection -# - url: https://www.wsj.com/science?mod=nav_top_subsection -# - url: https://www.wsj.com/science/archaeology?mod=nav_top_subsection -# - url: https://www.wsj.com/science/biology?mod=nav_top_subsection -# - url: https://www.wsj.com/science/environment?mod=nav_top_subsection -# - url: https://www.wsj.com/science/physics?mod=nav_top_subsection -# - url: https://www.wsj.com/science/space-astronomy?mod=nav_top_subsection -# - url: https://www.wsj.com/economy/central-banking?mod=nav_top_subsection -# - url: https://www.wsj.com/economy/consumers?mod=nav_top_subsection -# - url: https://www.wsj.com/economy/housing?mod=nav_top_subsection -# - url: https://www.wsj.com/economy/jobs?mod=nav_top_subsection -# - url: https://www.wsj.com/economy/trade?mod=nav_top_subsection -# - url: https://www.wsj.com/economy/global -# - url: https://www.wsj.com/tech/ai?mod=nav_top_subsection -# - url: https://www.wsj.com/tech/biotech -# - url: https://www.wsj.com/tech/cybersecurity?mod=nav_top_subsection -# - url: https://www.wsj.com/tech/personal-tech?mod=nav_top_subsection -# - url: https://www.reuters.com/ -# - url: https://www.reuters.com/business/aerospace-defense/ -# - url: https://www.reuters.com/business/autos-transportation/ -# - url: https://www.reuters.com/business/davos/ -# - url: https://www.reuters.com/business/energy/ -# - url: https://www.reuters.com/business/environment/ -# - url: https://www.reuters.com/business/finance/ -# - url: https://www.reuters.com/business/healthcare-pharmaceuticals/ -# - url: https://www.reuters.com/business/media-telecom/ -# - url: https://www.reuters.com/business/retail-consumer/ -# - url: https://www.reuters.com/business/future-of-health/ -# - url: https://www.reuters.com/business/future-of-money/ -# - url: https://www.reuters.com/business/take-five/ -# - url: https://www.reuters.com/business/world-at-work/ -# - url: https://www.reuters.com/breakingviews/ -# - url: https://www.reuters.com/technology/ -# - url: https://www.reuters.com/technology/cybersecurity/ -# - url: https://www.reuters.com/technology/space/ -# - url: https://www.reuters.com/technology/disrupted/ -# - url: https://www.reuters.com/technology/reuters-momentum/ -# - url: https://www.reuters.com/investigations/ -# - url: https://a16z.com/news-content/#latest -# - url: https://news.ycombinator.com/ -# - url: https://www.reddit.com/?rdt=48006 -# - url: https://news.crunchbase.com/ -# - url: https://www.cctv.com/ -# - url: https://sports.cctv.com/ From b5ba906b0973de6024d585d3ef03454b6d50b6ba Mon Sep 17 00:00:00 2001 From: Li Bo Date: Sat, 27 Jul 2024 23:22:03 +1000 Subject: [PATCH 10/36] Dev/onevision (#148) * feat: Add multi-choice parsing and processing functions This commit adds two new functions to the `egoschema/utils.py` file: `get_multi_choice_info` and `parse_multi_choice_response`. These functions are used to parse and process multi-choice responses in the Egoschema task. The `get_multi_choice_info` function extracts information about the available choices from the input document, while the `parse_multi_choice_response` function parses the generated response and returns the predicted index. These functions are essential for accurately processing multi-choice answers in the Egoschema task. * feat: Add regex-based parsing for multi-choice predictions This commit enhances the `perceptiontest_val_process_results_mc` function in the `utils.py` file. It introduces regex-based parsing to extract the predicted choice from the raw text prediction. If a match is found for A, B, C, or D, the matched letter is used as the prediction. Otherwise, an empty string is set as the prediction. This improvement ensures accurate processing of multi-choice predictions in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Improve accuracy calculation in perception test validation This commit refactors the `perceptiontest_val_aggregate_accuracy` function in the `utils.py` file. Instead of comparing the string representations of `answer_id` and `pred_id`, it now directly checks the `correct` field in the `accuracy` dictionary. This change ensures more accurate calculation of the overall accuracy in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * Refactor accuracy calculation in perception test validation * feat: Add SRT_API model to available models This commit adds the SRT_API model to the list of available models in the `__init__.py` file. This model can now be used for evaluation and testing purposes. Co-authored-by: [Co-author Name] <[coauthor@example.com]> --------- Co-authored-by: [Co-author Name] <[coauthor@example.com]> --- docs/commands.md | 25 +++ lmms_eval/models/__init__.py | 1 + lmms_eval/models/srt_api.py | 189 ++++++++++++++++++++ lmms_eval/tasks/egoschema/utils.py | 127 +++++++++---- lmms_eval/tasks/mmstar/mmstar.yaml | 3 + lmms_eval/tasks/mmstar/utils.py | 51 ++---- lmms_eval/tasks/perceptiontest/val/utils.py | 28 ++- 7 files changed, 351 insertions(+), 73 deletions(-) create mode 100755 lmms_eval/models/srt_api.py diff --git a/docs/commands.md b/docs/commands.md index 4f8c7a7df..bfe077a12 100755 --- a/docs/commands.md +++ b/docs/commands.md @@ -22,3 +22,28 @@ This mode supports a number of command-line arguments, the details of which can * `--limit` : Accepts an integer, or a float between 0.0 and 1.0 . If passed, will limit the number of documents to evaluate to the first X documents (if an integer) per task or first X% of documents per task. Useful for debugging, especially on costly API models. +## Usage with SRT API + +> install sglang + +```bash +git clone https://github.com/EvolvingLMMs-Lab/sglang.git +cd sglang +pip install -e "python[srt]" +``` + +> run sglang backend service with the following command + +```bash +python -m sglang.launch_server --model-path "\path\to\onevision" --tokenizer-path lmms-lab/llavanext-qwen-siglip-tokenizer --port=30000 --host=127.0.0.1 --tp-size=8 --chat-template=chatml-llava +``` + +You may need to install some dependencies for the above command to work (if you encounter some errors). + +```bash +pip install httpx==0.23.3 +pip install protobuf==3.20 +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ +``` + + diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py index d11149a93..d171f1082 100755 --- a/lmms_eval/models/__init__.py +++ b/lmms_eval/models/__init__.py @@ -37,6 +37,7 @@ "tinyllava": "TinyLlava", "llava_onevision": "LlavaOneVision", "llava_hf": "LlavaHf", + "srt_api": "SRT_API", "longva": "LongVA", "vila": "VILA", "xcomposer2d5": "XComposer2D5", diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py new file mode 100755 index 000000000..66ed7291c --- /dev/null +++ b/lmms_eval/models/srt_api.py @@ -0,0 +1,189 @@ +from accelerate import Accelerator, DistributedType +import base64 +from io import BytesIO +from copy import deepcopy +from decord import VideoReader, cpu +import numpy as np +from openai import OpenAI +from PIL import Image +import os +import json +from typing import List, Tuple +from tqdm import tqdm +import time + +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model + + +@register_model("srt_api") +class SRT_API(lmms): + def __init__( + self, + api_key: str = "EMPTY", + model_version: str = "default", + modality: str = "video", + host: str = "127.0.0.1", + port: int = 30000, + max_frames_num: int = 10, + timeout: int = 120, + continual_mode: bool = False, + response_persistent_folder: str = None, + **kwargs, + ) -> None: + super().__init__() + # Manually set a image token for GPT4V so that we can search for it + # and split the text and image + # Here we just use the same token as llava for convenient + self.model_version = model_version + self.modality = modality + self.max_frames_num = max_frames_num + self.image_token = "" + self.timeout = timeout + self.continual_mode = continual_mode + if self.continual_mode: + if response_persistent_folder is None: + raise ValueError("Continual mode requires a persistent path for the response. Please provide a valid path.") + + os.makedirs(response_persistent_folder, exist_ok=True) + self.response_persistent_folder = response_persistent_folder + self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json") + + if os.path.exists(self.response_persistent_file): + with open(self.response_persistent_file, "r") as f: + self.response_cache = json.load(f) + self.cache_mode = "resume" + else: + self.response_cache = {} + self.cache_mode = "start" + + accelerator = Accelerator() + self.client = OpenAI(api_key="EMPTY", base_url="http://127.0.0.1:30000/v1") + # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue." + if accelerator.num_processes > 1: + assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported." + self.accelerator = accelerator + if self.accelerator.is_local_main_process: + eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self.accelerator = accelerator + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + + self.device = self.accelerator.device + + # Function to encode the image + def encode_image(self, image: Image): + output_buffer = BytesIO() + image.save(output_buffer, format="PNG") + byte_data = output_buffer.getvalue() + base64_str = base64.b64encode(byte_data).decode("utf-8") + return base64_str + + # Function to encode the video + def encode_video(self, video_path, for_get_frames_num): + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + img = Image.fromarray(frame) + output_buffer = BytesIO() + img.save(output_buffer, format="PNG") + byte_data = output_buffer.getvalue() + base64_str = base64.b64encode(byte_data).decode("utf-8") + base64_frames.append(base64_str) + + return base64_frames + + def flatten(self, input): + new_list = [] + for i in input: + for j in i: + new_list.append(j) + return new_list + + def generate_until(self, requests) -> List[str]: + res = [] + pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding") + + for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]: + if self.continual_mode is True and self.cache_mode == "resume": + doc_uuid = f"{task}___{split}___{doc_id}" + if doc_uuid in self.response_cache: + response_text = self.response_cache[doc_uuid] + if response_text: + res.append(response_text) + pbar.update(1) + continue + + visuals = [doc_to_visual(self.task_dict[task][split][doc_id])] + visuals = self.flatten(visuals) + imgs = [] # multiple images or frames for video + for visual in visuals: + if self.modality == "image": + img = self.encode_image(visual) + imgs.append(img) + elif self.modality == "video": + frames = self.encode_video(visual, self.max_frames_num) + imgs.extend(frames) + + messages = [] + if self.image_token not in contexts: # single image format + content = [] + for img in imgs: + content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}) + + content.append({"type": "text", "text": contexts}) + messages.append({"role": "user", "content": content}) + else: # interleaved format + contexts = contexts.split(self.image_token) + for idx, img in enumerate(imgs): + content = [ + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}, + {"type": "text", "text": contexts[idx]}, + ] + messages.append({"role": "user", "content": content}) + messages.append({"role": "user", "content": [{"type": "text", "text": contexts[-1]}]}) + + if "max_new_tokens" not in gen_kwargs: + gen_kwargs["max_new_tokens"] = 1024 + + if "temperature" not in gen_kwargs: + gen_kwargs["temperature"] = 0 + + for attempt in range(5): + try: + response = self.client.chat.completions.create(model=self.model_version, messages=messages, temperature=gen_kwargs["temperature"], max_tokens=gen_kwargs["max_new_tokens"], timeout=self.timeout) + response_text = response.choices[0].message.content.strip() + break # If successful, break out of the loop + + except Exception as e: + eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}.") + if attempt < 4: + time.sleep(NUM_SECONDS_TO_SLEEP) + else: # If this was the last attempt, log and return empty string + eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}.") + response_text = "" + + res.append(response_text) + pbar.update(1) + + if self.continual_mode is True: # Cache the response + doc_uuid = f"{task}___{split}___{doc_id}" + self.response_cache[doc_uuid] = response_text + with open(self.response_persistent_file, "w") as f: + json.dump(self.response_cache, f) + + pbar.close() + return res + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + # TODO + assert False, "GPT4V not support" diff --git a/lmms_eval/tasks/egoschema/utils.py b/lmms_eval/tasks/egoschema/utils.py index 34d2ab588..d2c1f66c8 100755 --- a/lmms_eval/tasks/egoschema/utils.py +++ b/lmms_eval/tasks/egoschema/utils.py @@ -84,41 +84,106 @@ def egoschema_process_results(doc, result): return {"submission": {doc["video_idx"]: min_index}, "score": {"pred": min_index, "ground_truth": doc["answer"]}} +def get_multi_choice_info(doc): + all_choices = [] + index2ans = {} + OPTIONS = ["A", "B", "C", "D", "E"] + for i in range(5): + # import pdb;pdb.set_trace() + index2ans[OPTIONS[i]] = doc["option"][i].strip() + all_choices.append(OPTIONS[i]) + + return index2ans, all_choices + + +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10 + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + ans_with_space = False + ans_with_dot = False + candidates = [] + # import pdb; pdb.set_trace() + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(f"({choice})") + ans_with_brack = True + + # if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f"{choice} " in response: + candidates.append(f"{choice} ") + ans_with_space = True + + # if len(candidates) == 0: + for choice in all_choices: # e.g., A. B. C. D. + if f"{choice}." in response: + candidates.append(f"{choice}.") + ans_with_dot = True + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + # import pdb; pdb.set_trace() + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + # candidates = list(set(candidates)) + start_indexes = [] + if index_ans: + # if ans_with_brack: + for can in candidates: + index = response.rfind(can) + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + # if ans_with_space: + # for can in candidates: + # index = response.rfind(f"{can} ") + # start_indexes.append(index) + # if ans_with_dot: + # for can in candidates: + # index = response.rfind(f"{can}.") + # start_indexes.append(index) + # if not ans_with_brack and not ans_with_space and not ans_with_dot: + # for can in candidates: + # index = response.rfind(f" {can} ") + # start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the first one + pred_index = candidates[np.argmin(start_indexes)] + pred_index = pred_index.replace("(", "").replace(")", "").replace(".", "").strip() + else: # if only one candidate, use it. + pred_index = candidates[0] + pred_index = pred_index.replace("(", "").replace(")", "").replace(".", "").strip() + + return pred_index, len(candidates) > 0 + + # Process result for mcq answer generation def egoschema_process_results_generation(doc, result): + # import pdb;pdb.set_trace() pred = result[0] - # Determine whether the video LLM output is correct, based on word matching rules - # Ensure each option string ends with a period - option_strs = [opt if opt.endswith(".") else opt + "." for opt in doc["option"]] # Complete option strings - option_sents = [opt.split(". ")[1] if ". " in opt else opt for opt in option_strs] # Option sentence - option_inds = [opt.split(". ")[0] if ". " in opt else opt for opt in option_strs] # Option letter, e.g., A, B, C, D, E - - video_llm_pred = None - index = -1 - - # Check if the prediction matches any of the complete option strings - for idx, option_str in enumerate(option_strs): - if pred == option_str: - video_llm_pred = option_str - index = idx - break - - # Check if the prediction matches any of the option sentences - if not video_llm_pred: - for idx, option_sent in enumerate(option_sents): - if pred == option_sent: - video_llm_pred = option_sent - index = idx - break - - # Check if the prediction matches any of the option letters - if not video_llm_pred: - for idx, option_ind in enumerate(option_inds): - if pred == option_ind or pred == option_ind.replace(".", ""): - video_llm_pred = option_ind - index = idx - break + index2ans, all_choices = get_multi_choice_info(doc) + parsed_pred, matched_tag = parse_multi_choice_response(pred, all_choices, index2ans) + + pred_to_index = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4} + index = pred_to_index.get(parsed_pred, -1) # Default to -1 if the prediction is not found return {"submission": {doc["video_idx"]: index}, "score": {"pred": index, "ground_truth": doc["answer"]}} diff --git a/lmms_eval/tasks/mmstar/mmstar.yaml b/lmms_eval/tasks/mmstar/mmstar.yaml index 845978460..338efe2f4 100644 --- a/lmms_eval/tasks/mmstar/mmstar.yaml +++ b/lmms_eval/tasks/mmstar/mmstar.yaml @@ -29,6 +29,9 @@ metric_list: - metric: math aggregation: !function utils.mmstar_aggregate_results higher_is_better: true + - metric: average + aggregation: !function utils.mmstar_aggregate_results + higher_is_better: true model_specific_prompt_kwargs: default: pre_prompt: "" diff --git a/lmms_eval/tasks/mmstar/utils.py b/lmms_eval/tasks/mmstar/utils.py index 66933f299..b1b2c6f02 100644 --- a/lmms_eval/tasks/mmstar/utils.py +++ b/lmms_eval/tasks/mmstar/utils.py @@ -10,36 +10,12 @@ dir_name = os.path.dirname(os.path.abspath(__file__)) eval_type_dict = { - "coarse perception" : [ - "image scene and topic", - "image style & quality", - "image emotion" - ], - "fine-grained perception" : [ - "object counting", - "recognition", - "localization" - ], - "instance reasoning" : [ - "single-instance reasoning", - "cross-instance attribute reasoning", - "cross-instance relation reasoning" - ], - "logical reasoning" : [ - "code & sequence reasoning", - "diagram reasoning", - "common reasoning" - ], - "science & technology" : [ - "biology & chemistry & physics", - "electronics & energy & mechanical eng.", - "geography & earth science & agriculture" - ], - "math" : [ - "geometry", - "numeric commonsense and calculation", - "statistical reasoning" - ] + "coarse perception": ["image scene and topic", "image style & quality", "image emotion"], + "fine-grained perception": ["object counting", "recognition", "localization"], + "instance reasoning": ["single-instance reasoning", "cross-instance attribute reasoning", "cross-instance relation reasoning"], + "logical reasoning": ["code & sequence reasoning", "diagram reasoning", "common reasoning"], + "science & technology": ["biology & chemistry & physics", "electronics & energy & mechanical eng.", "geography & earth science & agriculture"], + "math": ["geometry", "numeric commonsense and calculation", "statistical reasoning"], } @@ -63,16 +39,16 @@ def mmstar_doc_to_text(doc, model_specific_prompt_kwargs=None): def exact_match(pred, gt): """Brought from MMStar""" - answer = gt.lower().strip().replace('\n', ' ') - predict = pred.lower().strip().replace('\n', ' ') + answer = gt.lower().strip().replace("\n", " ") + predict = pred.lower().strip().replace("\n", " ") try: if answer == predict[0]: return 1.0 - elif predict[0] == '(' and answer == predict[1]: + elif predict[0] == "(" and answer == predict[1]: return 1.0 - elif predict[0:7] == 'option ' and answer == predict[7]: + elif predict[0:7] == "option " and answer == predict[7]: return 1.0 - elif predict[0:14] == 'the answer is ' and answer == predict[14]: + elif predict[0:14] == "the answer is " and answer == predict[14]: return 1.0 except Exception as e: return 0.0 @@ -93,7 +69,7 @@ def mmstar_process_results(doc, results): score = exact_match(pred, gt) category = doc["category"] l2_category = doc["l2_category"] - return {category: {"question_id": doc["index"], "l2_category": l2_category, "score": score}} + return {category: {"question_id": doc["index"], "l2_category": l2_category, "score": score}, "average": {"question_id": doc["index"], "l2_category": l2_category, "score": score}} def mmstar_aggregate_results(results): @@ -114,7 +90,6 @@ def mmstar_aggregate_results(results): avg_score = sum(scores) / len(scores) l2_category_avg_score[l2_category] = avg_score eval_logger.info(f"{l2_category}: {avg_score:.2f}") - + avg_score = sum(l2_category_avg_score.values()) / len(l2_category_avg_score) return avg_score - \ No newline at end of file diff --git a/lmms_eval/tasks/perceptiontest/val/utils.py b/lmms_eval/tasks/perceptiontest/val/utils.py index 562cc0fe1..63156fe28 100755 --- a/lmms_eval/tasks/perceptiontest/val/utils.py +++ b/lmms_eval/tasks/perceptiontest/val/utils.py @@ -103,15 +103,35 @@ def perceptiontest_val_process_results_mc_ppl(doc, result): # Process result for generation +import re + + def perceptiontest_val_process_results_mc(doc, result): - pred = result[0] # string prediction "A", "B", "C" + pred = result[0].strip() # raw text prediction + + # Use regex to match A, B, C, or D + match = re.search(r"\b([A-D])\b", pred) + + if match: + pred = match.group(1) # Extract the matched letter + pred = pred.upper() + else: + pred = "" # Set to empty string if no match found # Map the prediction to an index - pred_to_index = {"A": 0, "B": 1, "C": 2} + pred_to_index = {"A": 0, "B": 1, "C": 2, "D": 3} index = pred_to_index.get(pred, -1) # Default to -1 if the prediction is not found + correct = 1 if index == int(doc["answer_id"]) else 0 + return { - "accuracy": {"video_name": doc["video_name"], "question": doc["question"], "question_id": doc["question_id"], "pred_id": index, "answer_id": doc["answer_id"], "area": doc["area"], "reasoning": doc["reasoning"], "tag": doc["tag"]} + "accuracy": { + "video_name": doc["video_name"], + "question": doc["question"], + "pred_id": index, + "answer_id": doc["answer_id"], + "correct": correct, + } } @@ -120,7 +140,7 @@ def perceptiontest_val_aggregate_accuracy(results, args): # results is a list of dict for answer_dict in results: - if str(answer_dict["answer_id"]) == str(answer_dict["pred_id"]): + if answer_dict["correct"] == 1: yes_count = yes_count + 1 accuracy = yes_count / len(results) From 96a6732f3c93f913bacafff8613d4462f43cd1af Mon Sep 17 00:00:00 2001 From: Li Bo Date: Sat, 27 Jul 2024 23:23:22 +1000 Subject: [PATCH 11/36] Update commands.md --- docs/commands.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/commands.md b/docs/commands.md index bfe077a12..24b7a0aa0 100755 --- a/docs/commands.md +++ b/docs/commands.md @@ -35,7 +35,11 @@ pip install -e "python[srt]" > run sglang backend service with the following command ```bash +# backend service python -m sglang.launch_server --model-path "\path\to\onevision" --tokenizer-path lmms-lab/llavanext-qwen-siglip-tokenizer --port=30000 --host=127.0.0.1 --tp-size=8 --chat-template=chatml-llava + +# launch lmms-eval srt_api model +python -m accelerate.commands.launch --main_process_port=12580 --num_processes=1 lmms_eval --model=srt_api --model_args=modality=image,host=127.0.0.1,port=30000 --tasks=ai2d --batch_size=1 --log_samples --log_samples_suffix=debug --output_path=./logs/ --verbosity=DEBUG ``` You may need to install some dependencies for the above command to work (if you encounter some errors). From 1537ce61b3c5f0194116f5199818126dac146192 Mon Sep 17 00:00:00 2001 From: Li Bo Date: Sun, 28 Jul 2024 00:04:16 +1000 Subject: [PATCH 12/36] Update commands.md --- docs/commands.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/commands.md b/docs/commands.md index 24b7a0aa0..e303fc644 100755 --- a/docs/commands.md +++ b/docs/commands.md @@ -28,7 +28,8 @@ This mode supports a number of command-line arguments, the details of which can ```bash git clone https://github.com/EvolvingLMMs-Lab/sglang.git -cd sglang +cd sglang; +git checkout dev/onevision; pip install -e "python[srt]" ``` From 8d6d8080caceefb604266852af43f1f41c18df2d Mon Sep 17 00:00:00 2001 From: kcz358 Date: Mon, 29 Jul 2024 07:22:56 +0000 Subject: [PATCH 13/36] Fix internvl2 bugs --- lmms_eval/models/internvl2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lmms_eval/models/internvl2.py b/lmms_eval/models/internvl2.py index 4fdad818e..c32258422 100644 --- a/lmms_eval/models/internvl2.py +++ b/lmms_eval/models/internvl2.py @@ -136,8 +136,8 @@ def __init__( super().__init__() self.path = pretrained - self.model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True).eval().cuda() - self.tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True) + self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, trust_remote_code=True).eval().cuda() + self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True) batch_size = int(batch_size) assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}." From 9c2467996f85f7de8c051fdf063e1dc8b4d71288 Mon Sep 17 00:00:00 2001 From: kcz358 Date: Mon, 29 Jul 2024 07:27:33 +0000 Subject: [PATCH 14/36] Set num_threads to 1 for VideoReader in srt --- lmms_eval/models/srt_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py index 66ed7291c..a5e4a0bd2 100755 --- a/lmms_eval/models/srt_api.py +++ b/lmms_eval/models/srt_api.py @@ -85,7 +85,7 @@ def encode_image(self, image: Image): # Function to encode the video def encode_video(self, video_path, for_get_frames_num): - vr = VideoReader(video_path, ctx=cpu(0)) + vr = VideoReader(video_path, ctx=cpu(0), num_threads=1) total_frame_num = len(vr) uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int) frame_idx = uniform_sampled_frames.tolist() From 5d9e5f6e4112a475cacbd018013767c55f43cf5c Mon Sep 17 00:00:00 2001 From: kcz358 Date: Mon, 29 Jul 2024 12:16:23 +0000 Subject: [PATCH 15/36] Use pyav to decode video when decord can not load, add video loading exceptions handling --- lmms_eval/models/srt_api.py | 45 +++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py index a5e4a0bd2..239ec4cf9 100755 --- a/lmms_eval/models/srt_api.py +++ b/lmms_eval/models/srt_api.py @@ -16,6 +16,8 @@ from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model +from loguru import logger as eval_logger + @register_model("srt_api") class SRT_API(lmms): @@ -85,11 +87,27 @@ def encode_image(self, image: Image): # Function to encode the video def encode_video(self, video_path, for_get_frames_num): - vr = VideoReader(video_path, ctx=cpu(0), num_threads=1) - total_frame_num = len(vr) - uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int) - frame_idx = uniform_sampled_frames.tolist() - frames = vr.get_batch(frame_idx).asnumpy() + try: + vr = VideoReader(video_path, ctx=cpu(0), num_threads=1) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, for_get_frames_num, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + except: + import av + container = av.open(video_path) + + frames = [] + # https://github.com/PyAV-Org/PyAV/issues/1269 + # https://www.cnblogs.com/beyond-tester/p/17641872.html + # context = CodecContext.create("libvpx-vp9", "r") + for packet in container.demux(video=0): + for frame in packet.decode(): + frames.append(frame) + total_frames = len(frames) + sampled_frm = min(total_frames, for_get_frames_num) + indices = np.linspace(0, total_frames - 1, sampled_frm, dtype=int) + frames = [frames[i] for i in indices] base64_frames = [] for frame in frames: @@ -131,8 +149,21 @@ def generate_until(self, requests) -> List[str]: img = self.encode_image(visual) imgs.append(img) elif self.modality == "video": - frames = self.encode_video(visual, self.max_frames_num) - imgs.extend(frames) + try: + frames = self.encode_video(visual, self.max_frames_num) + imgs.extend(frames) + except Exception as e: + eval_logger.error(f"Exception : {e} \n When loading video {visual}") + imgs = None + break + + # Handling video decode error + # If we can't even load using pyav, then we will skip + if imgs is None: + resps = "" + res.append(resps) + pbar.update(1) + continue messages = [] if self.image_token not in contexts: # single image format From 2ddedaf69da52452ba61ee3919bdf943e882f2f5 Mon Sep 17 00:00:00 2001 From: Li Bo Date: Tue, 30 Jul 2024 10:43:20 +1000 Subject: [PATCH 16/36] Dev/onevision (#149) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task * Update lmms_eval configuration to use process_results_use_image and full_docs options * split tasks * chore: Update live_bench task dataset names and versions * feat: Add multi-choice parsing and processing functions This commit adds two new functions to the `egoschema/utils.py` file: `get_multi_choice_info` and `parse_multi_choice_response`. These functions are used to parse and process multi-choice responses in the Egoschema task. The `get_multi_choice_info` function extracts information about the available choices from the input document, while the `parse_multi_choice_response` function parses the generated response and returns the predicted index. These functions are essential for accurately processing multi-choice answers in the Egoschema task. * feat: Add regex-based parsing for multi-choice predictions This commit enhances the `perceptiontest_val_process_results_mc` function in the `utils.py` file. It introduces regex-based parsing to extract the predicted choice from the raw text prediction. If a match is found for A, B, C, or D, the matched letter is used as the prediction. Otherwise, an empty string is set as the prediction. This improvement ensures accurate processing of multi-choice predictions in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Improve accuracy calculation in perception test validation This commit refactors the `perceptiontest_val_aggregate_accuracy` function in the `utils.py` file. Instead of comparing the string representations of `answer_id` and `pred_id`, it now directly checks the `correct` field in the `accuracy` dictionary. This change ensures more accurate calculation of the overall accuracy in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * Refactor accuracy calculation in perception test validation * feat: Add SRT_API model to available models This commit adds the SRT_API model to the list of available models in the `__init__.py` file. This model can now be used for evaluation and testing purposes. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * chore: Update live_bench task dataset names and versions (cherry picked from commit 46a85b40b013503e52d007c96ca0607bd4604a3e) * refactor: Update question_for_eval key in MathVerseEvaluator This commit updates the key "question_for_eval" to "question" in the MathVerseEvaluator class. This change ensures consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update generate_submission_file function in mathverse utils This commit updates the `generate_submission_file` function in the `mathverse/utils.py` file. It adds the `problem_version` to the filename to ensure that the submission files are saved with the correct problem version. This change improves the organization and clarity of the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files This commit updates the default template YAML files in the `lmms_eval/tasks` directory. It modifies the `generation_kwargs` and `metadata` sections to improve the configuration and versioning of the tasks. These changes ensure consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> --------- Co-authored-by: Fanyi Pu Co-authored-by: [Co-author Name] <[coauthor@example.com]> Co-authored-by: kcz358 --- lmms_eval/api/task.py | 2 + lmms_eval/evaluator.py | 7 +--- .../_default_template_internal_eval_yaml | 3 +- lmms_eval/tasks/live_bench/live_bench.yaml | 33 +++------------ .../tasks/live_bench/live_bench_2406.yaml | 3 ++ .../tasks/live_bench/live_bench_2407.yaml | 3 ++ .../tasks/live_bench/live_bench_template_yaml | 28 +++++++++++++ .../_default_template_wilder_yaml | 1 + lmms_eval/tasks/mathverse/mathverse_evals.py | 2 +- .../mathverse/mathverse_testmini_vision.yaml | 10 +++++ lmms_eval/tasks/mathverse/utils.py | 7 ++-- lmms_eval/tasks/mmmu/_default_template_yaml | 6 +++ lmms_eval/tasks/mmmu/mmmu_test.yaml | 9 ++-- lmms_eval/tasks/mmmu/mmmu_val.yaml | 11 ++--- lmms_eval/tasks/mmmu/utils.py | 27 ++++++++---- lmms_eval/tasks/nextqa/_default_template_yaml | 3 ++ lmms_eval/tasks/nextqa/utils.py | 42 ++++++++----------- .../videochatgpt_consistency.yaml | 4 +- .../videochatgpt/videochatgpt_generic.yaml | 7 ++++ .../videochatgpt/videochatgpt_temporal.yaml | 3 -- .../wild_vision_bench/_default_template_yaml | 1 + 21 files changed, 122 insertions(+), 90 deletions(-) mode change 100644 => 100755 lmms_eval/tasks/live_bench/live_bench.yaml create mode 100644 lmms_eval/tasks/live_bench/live_bench_2406.yaml create mode 100644 lmms_eval/tasks/live_bench/live_bench_2407.yaml create mode 100644 lmms_eval/tasks/live_bench/live_bench_template_yaml create mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_vision.yaml create mode 100644 lmms_eval/tasks/mmmu/_default_template_yaml diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index 5560745c0..f488bb224 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -67,8 +67,10 @@ class TaskConfig(dict): validation_split: str = None test_split: str = None fewshot_split: str = None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaling (?) + full_docs: bool = False # formatting / prompting options. # see docs/advanced_task_guide.md for more info + process_results_use_image: bool = False process_docs: Callable = None doc_to_visual: Union[Callable, str] = None doc_to_text: Union[Callable, str] = None diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py index 48397a0a2..0e890533f 100755 --- a/lmms_eval/evaluator.py +++ b/lmms_eval/evaluator.py @@ -327,7 +327,7 @@ def evaluate( # hack: remove image columns to speed avoid loading images and speed up postprocessing # reason: doc_iterator will actually load image if it's in the doc. docs = task.test_docs() if task.has_test_docs() else task.validation_docs() - if "d170" not in task_name and "dc100" not in task_name and "dc200" not in task_name and "llava_wilder" not in task_name and "live_bench" not in task_name and "wildvision" not in task_name: + if not task.config["process_results_use_image"]: remove_cols = [] features = docs.features # If it is an Image instance or a Sequence of Image instance. Remove it @@ -340,10 +340,7 @@ def evaluate( docs = docs.remove_columns(remove_cols) ####################### Processing with Full Docs Mode ####################### - if task_name in ["videochatgpt_consistency"]: - full_docs = True - else: - full_docs = False + full_docs = task.config["full_docs"] doc_iterator = itertools.islice(enumerate(docs), lm.rank, limit, lm.world_size) # Instead of converting the iterator to a list, use `itertools.tee` to create a parallel iterator for counting diff --git a/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml b/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml index cd692a1d8..5281ad1de 100755 --- a/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml +++ b/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml @@ -1,4 +1,5 @@ model_specific_prompt_kwargs: default: pre_prompt: "" - post_prompt: "" \ No newline at end of file + post_prompt: "" +process_results_use_image: true diff --git a/lmms_eval/tasks/live_bench/live_bench.yaml b/lmms_eval/tasks/live_bench/live_bench.yaml old mode 100644 new mode 100755 index b4a09853d..1253105ee --- a/lmms_eval/tasks/live_bench/live_bench.yaml +++ b/lmms_eval/tasks/live_bench/live_bench.yaml @@ -1,31 +1,8 @@ -dataset_path: lmms-lab/LiveBench -dataset_kwargs: - token: True -task: "live_bench" -test_split: test -dataset_name: 2024-07 -output_type: generate_until -doc_to_visual: !function utils.livebench_doc_to_visual -doc_to_text: !function utils.livebench_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 1024 - temperature: 0 - top_p: 1.0 - num_beams: 1 - do_sample: false -process_results: !function utils.livebench_process_results -metric_list: - - metric: gpt4_eval_score - aggregation: !function utils.livebench_aggregate_results - higher_is_better: true - # - metric: gpt4_eval_score_mini - # aggregation: !function utils.livebench_aggregate_results - # higher_is_better: true -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" +group: live_bench +task: +- live_bench_2406 +- live_bench_2407 + metadata: api_type : openai eval_with_mini: false diff --git a/lmms_eval/tasks/live_bench/live_bench_2406.yaml b/lmms_eval/tasks/live_bench/live_bench_2406.yaml new file mode 100644 index 000000000..c82eba4c5 --- /dev/null +++ b/lmms_eval/tasks/live_bench/live_bench_2406.yaml @@ -0,0 +1,3 @@ +task: "live_bench_2406" +dataset_name: 2024-06 +include: live_bench_template_yaml diff --git a/lmms_eval/tasks/live_bench/live_bench_2407.yaml b/lmms_eval/tasks/live_bench/live_bench_2407.yaml new file mode 100644 index 000000000..d77913454 --- /dev/null +++ b/lmms_eval/tasks/live_bench/live_bench_2407.yaml @@ -0,0 +1,3 @@ +task: "live_bench_2407" +dataset_name: 2024-07 +include: live_bench_template_yaml diff --git a/lmms_eval/tasks/live_bench/live_bench_template_yaml b/lmms_eval/tasks/live_bench/live_bench_template_yaml new file mode 100644 index 000000000..e3d9877ed --- /dev/null +++ b/lmms_eval/tasks/live_bench/live_bench_template_yaml @@ -0,0 +1,28 @@ +dataset_path: lmms-lab/LiveBench +dataset_kwargs: + token: True +test_split: test +dataset_name: 2024-07 +output_type: generate_until +doc_to_visual: !function utils.livebench_doc_to_visual +doc_to_text: !function utils.livebench_doc_to_text +doc_to_target: "answer" +generation_kwargs: + max_new_tokens: 1024 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false +process_results: !function utils.livebench_process_results +process_results_use_image: true +metric_list: + - metric: gpt4_eval_score + aggregation: !function utils.livebench_aggregate_results + higher_is_better: true + # - metric: gpt4_eval_score_mini + # aggregation: !function utils.livebench_aggregate_results + # higher_is_better: true +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "" diff --git a/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml b/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml index 2484f795c..e18ee148e 100644 --- a/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml +++ b/lmms_eval/tasks/llava_wilder/_default_template_wilder_yaml @@ -9,6 +9,7 @@ generation_kwargs: num_beams: 1 do_sample: false process_results: !function utils.llava_process_results +process_results_use_image: true metric_list: - metric: gpt_eval_llava_all aggregation: !function utils.llava_all_aggregation diff --git a/lmms_eval/tasks/mathverse/mathverse_evals.py b/lmms_eval/tasks/mathverse/mathverse_evals.py index 5894f6f70..fb72519b6 100644 --- a/lmms_eval/tasks/mathverse/mathverse_evals.py +++ b/lmms_eval/tasks/mathverse/mathverse_evals.py @@ -265,7 +265,7 @@ def eval_results(self, results, config): problem = { "question_type": inst["question_type"], "answer": inst["answer"] if "answer" in inst else None, - "question_for_eval": inst["question_for_eval"], + "question_for_eval": inst["question"], } if config["metadata"].get("trunk_response", -1) > 0: prediction = " ".join(full_prediction.split(" ")[-config["metadata"]["trunk_response"] :]) diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision.yaml new file mode 100644 index 000000000..261a23f40 --- /dev/null +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_vision.yaml @@ -0,0 +1,10 @@ +group: mathverse_testmini_vision +task: + - mathverse_testmini_vision_intensive + - mathverse_testmini_vision_dominant + - mathverse_testmini_vision_only +metadata: + version: 0.0 + gpt_eval_model_name: "gpt-3.5-turbo" + trunk_response: 30 + quick_match: false \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/utils.py b/lmms_eval/tasks/mathverse/utils.py index de22d93d3..4ff81ff03 100644 --- a/lmms_eval/tasks/mathverse/utils.py +++ b/lmms_eval/tasks/mathverse/utils.py @@ -75,18 +75,19 @@ def mathverse_aggregate_results_submission(results, args, *, calculate_gain=Fals def mathverse_aggregate_results_eval(results, args, *, calculate_gain=False, random_scores=None): split_flag = results[0]["metadata"]["split"] + problem_version = results[0]["metadata"]["problem_version"].lower().replace(" ", "_") # save the result first, in case the gpt evaluation fails - path = generate_submission_file(f"mathverse_{split_flag}_results.json", args) + path = generate_submission_file(f"mathverse_{split_flag}_{problem_version}_results.json", args) with open(path, "w") as f: json.dump(results, f, indent=4) # gpt evaluation results_dict, scores = mathverse_evaluator.eval_results(results, config) # save results - path = generate_submission_file(f"mathverse_{split_flag}_results.json", args) + path = generate_submission_file(f"mathverse_{split_flag}_{problem_version}_results.json", args) with open(path, "w") as f: json.dump(results_dict, f, indent=4) # save scores - path = generate_submission_file(f"mathverse_{split_flag}_scores.json", args) + path = generate_submission_file(f"mathverse_{split_flag}_{problem_version}_scores.json", args) with open(path, "w") as f: json.dump(scores, f, indent=4) eval_logger.info(f"Saved scores to {path}") diff --git a/lmms_eval/tasks/mmmu/_default_template_yaml b/lmms_eval/tasks/mmmu/_default_template_yaml new file mode 100644 index 000000000..a53675347 --- /dev/null +++ b/lmms_eval/tasks/mmmu/_default_template_yaml @@ -0,0 +1,6 @@ +generation_kwargs: + max_new_tokens: 16 + +metadata: + version: 0.0 + interleaved_format: false \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/mmmu_test.yaml b/lmms_eval/tasks/mmmu/mmmu_test.yaml index 8f1a1f204..03564b6a7 100755 --- a/lmms_eval/tasks/mmmu/mmmu_test.yaml +++ b/lmms_eval/tasks/mmmu/mmmu_test.yaml @@ -7,13 +7,10 @@ doc_to_text: !function utils.mmmu_doc_to_text doc_to_target: "answer" # The return value of process_results will be used by metrics process_results: !function utils.mmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: original + metric_list: - metric: submission aggregation: !function utils.mmmu_test_aggregate_results_for_submission higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/mmmu_val.yaml b/lmms_eval/tasks/mmmu/mmmu_val.yaml index 9e1574c43..a301f7cb8 100755 --- a/lmms_eval/tasks/mmmu/mmmu_val.yaml +++ b/lmms_eval/tasks/mmmu/mmmu_val.yaml @@ -7,15 +7,10 @@ doc_to_text: !function utils.mmmu_doc_to_text doc_to_target: "answer" # The return value of process_results will be used by metrics process_results: !function utils.mmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 128 -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original + metric_list: - metric: mmmu_acc aggregation: !function utils.mmmu_aggregate_results higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/utils.py b/lmms_eval/tasks/mmmu/utils.py index 83aeed200..ffa89c3f6 100755 --- a/lmms_eval/tasks/mmmu/utils.py +++ b/lmms_eval/tasks/mmmu/utils.py @@ -5,7 +5,8 @@ import numpy as np import os import json - +from pathlib import Path +import yaml from lmms_eval.tasks._task_utils.file_utils import generate_submission_file @@ -14,13 +15,23 @@ MULTI_CHOICE_PROMPT = "Answer with the option's letter from the given choices directly." OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase." +with open(Path(__file__).parent / "_default_template_yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + def replace_images_tokens(input_string): - # for i in range(1, 8): - # question_text = f"" - # query_text = "" - # if question_text in input_string: - # input_string = input_string.replace(question_text, query_text) + for i in range(1, 8): + question_text = f"" + query_text = "" + if question_text in input_string: + input_string = input_string.replace(question_text, query_text) return input_string @@ -44,7 +55,9 @@ def construct_prompt(doc): def mmmu_doc_to_text(doc): question = construct_prompt(doc) - return replace_images_tokens(question) + if config["metadata"]["interleaved_format"]: + question = replace_images_tokens(question) + return question def mmmu_doc_to_visual(doc): diff --git a/lmms_eval/tasks/nextqa/_default_template_yaml b/lmms_eval/tasks/nextqa/_default_template_yaml index 65f3845a6..e30dd25fe 100644 --- a/lmms_eval/tasks/nextqa/_default_template_yaml +++ b/lmms_eval/tasks/nextqa/_default_template_yaml @@ -3,3 +3,6 @@ dataset_kwargs: token: True video: True cache_dir: nextqa +metadata: + version: 0.0.1 + load_package: False \ No newline at end of file diff --git a/lmms_eval/tasks/nextqa/utils.py b/lmms_eval/tasks/nextqa/utils.py index 4fa465232..6556723b7 100644 --- a/lmms_eval/tasks/nextqa/utils.py +++ b/lmms_eval/tasks/nextqa/utils.py @@ -1,40 +1,15 @@ import os import yaml - import random import pandas as pd - from pathlib import Path - from loguru import logger as eval_logger -try: - from pywsd.utils import lemmatize_sentence -except ImportError: - eval_logger.debug("pywsd not installed. Please install pywsd to use this module. You can install it by running 'pip install pywsd'") - -try: - from nltk.tokenize import word_tokenize - from nltk.corpus import wordnet - - try: - import nltk - - nltk.download("averaged_perceptron_tagger", quiet=True) - nltk.download("wordnet", quiet=True) - nltk.download("punkt", quiet=True) - except Exception as e: - eval_logger.debug(f"nltk download failed: {e}") -except ImportError: - eval_logger.debug("nltk not installed. Please install nltk to use this module. You can install it by running 'pip install nltk'") - from lmms_eval.tasks._task_utils.video_loader import get_cache_dir, get_video import numpy as np - OPTIONS = ["A", "B", "C", "D", "E"] - with open(Path(__file__).parent / "_default_template_yaml", "r") as f: raw_data = f.readlines() safe_data = [] @@ -45,6 +20,23 @@ config = yaml.safe_load("".join(safe_data)) +if config["metadata"]["load_package"]: + try: + from pywsd.utils import lemmatize_sentence + except ImportError: + eval_logger.debug("pywsd not installed. Please install pywsd to use this module. You can install it by running 'pip install pywsd'") + + try: + from nltk.tokenize import word_tokenize + from nltk.corpus import wordnet + import nltk + + nltk.download("averaged_perceptron_tagger", quiet=True) + nltk.download("wordnet", quiet=True) + nltk.download("punkt", quiet=True) + except ImportError: + eval_logger.debug("nltk not installed. Please install nltk to use this module. You can install it by running 'pip install nltk'") + stopwords = set(pd.read_csv(Path(__file__).parent / "stopwords.csv").squeeze()) cache_dir = get_cache_dir(config, "NExTVideo") diff --git a/lmms_eval/tasks/videochatgpt/videochatgpt_consistency.yaml b/lmms_eval/tasks/videochatgpt/videochatgpt_consistency.yaml index 936878bfe..f5562bf3e 100755 --- a/lmms_eval/tasks/videochatgpt/videochatgpt_consistency.yaml +++ b/lmms_eval/tasks/videochatgpt/videochatgpt_consistency.yaml @@ -11,11 +11,9 @@ metric_list: aggregation: !function utils.videochatgpt_aggregate_consistency higher_is_better: true include: _default_template_yaml +full_docs: true generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original max_new_tokens: 1024 temperature: 0 top_p: 1.0 diff --git a/lmms_eval/tasks/videochatgpt/videochatgpt_generic.yaml b/lmms_eval/tasks/videochatgpt/videochatgpt_generic.yaml index 9affe5340..4214eefd4 100755 --- a/lmms_eval/tasks/videochatgpt/videochatgpt_generic.yaml +++ b/lmms_eval/tasks/videochatgpt/videochatgpt_generic.yaml @@ -17,3 +17,10 @@ metric_list: aggregation: !function utils.videochatgpt_aggregate_score higher_is_better: true include: _default_template_yaml + +generation_kwargs: + max_new_tokens: 1024 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false diff --git a/lmms_eval/tasks/videochatgpt/videochatgpt_temporal.yaml b/lmms_eval/tasks/videochatgpt/videochatgpt_temporal.yaml index 1e207c2af..54bb57a9c 100755 --- a/lmms_eval/tasks/videochatgpt/videochatgpt_temporal.yaml +++ b/lmms_eval/tasks/videochatgpt/videochatgpt_temporal.yaml @@ -13,9 +13,6 @@ metric_list: include: _default_template_yaml generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original max_new_tokens: 1024 temperature: 0 top_p: 1.0 diff --git a/lmms_eval/tasks/wild_vision_bench/_default_template_yaml b/lmms_eval/tasks/wild_vision_bench/_default_template_yaml index 7ce709dc5..33d18d393 100644 --- a/lmms_eval/tasks/wild_vision_bench/_default_template_yaml +++ b/lmms_eval/tasks/wild_vision_bench/_default_template_yaml @@ -5,6 +5,7 @@ output_type: generate_until doc_to_visual: !function utils.wild_vision_doc_to_visual doc_to_text: !function utils.wild_vision_doc_to_text doc_to_target: !function utils.wild_vision_doc_to_target +process_results_use_image: true generation_kwargs: max_new_tokens: 4096 temperature: 0 From 9b558d7b213b8f0b3022d5f8eb54644690b6cd5a Mon Sep 17 00:00:00 2001 From: kcz358 Date: Tue, 30 Jul 2024 06:43:33 +0000 Subject: [PATCH 17/36] Fix mathverse error --- lmms_eval/tasks/mathverse/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lmms_eval/tasks/mathverse/utils.py b/lmms_eval/tasks/mathverse/utils.py index 4ff81ff03..dde6eeb27 100644 --- a/lmms_eval/tasks/mathverse/utils.py +++ b/lmms_eval/tasks/mathverse/utils.py @@ -65,7 +65,11 @@ def mathverse_process_results(doc, results): def mathverse_aggregate_results_submission(results, args, *, calculate_gain=False, random_scores=None): - split_flag = results[0]["metadata"]["split"] + # Don't know why but this sometimes yields error so I hardcode it + try: + split_flag = results[0]["metadata"]["split"] + except: + split_flag = "testmini" path = generate_submission_file(f"mathverse_{split_flag}_results.json", args) with open(path, "w") as f: json.dump(results, f, indent=4) @@ -75,7 +79,7 @@ def mathverse_aggregate_results_submission(results, args, *, calculate_gain=Fals def mathverse_aggregate_results_eval(results, args, *, calculate_gain=False, random_scores=None): split_flag = results[0]["metadata"]["split"] - problem_version = results[0]["metadata"]["problem_version"].lower().replace(" ", "_") + problem_version = results[0]["problem_version"].lower().replace(" ", "_") # save the result first, in case the gpt evaluation fails path = generate_submission_file(f"mathverse_{split_flag}_{problem_version}_results.json", args) with open(path, "w") as f: From 4d785c994961263eb1c82d308cfa3079cf023994 Mon Sep 17 00:00:00 2001 From: kcz358 Date: Thu, 1 Aug 2024 03:20:34 +0000 Subject: [PATCH 18/36] Add run examples doc --- docs/run_examples.md | 442 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 docs/run_examples.md diff --git a/docs/run_examples.md b/docs/run_examples.md new file mode 100644 index 000000000..91723b060 --- /dev/null +++ b/docs/run_examples.md @@ -0,0 +1,442 @@ +# User Guide +This document details the running examples for different models in `lmms_eval`. We include commandas on how to prepare environments for different model and some commands to run these models + +## Environmental Variables + +Before running experiments and evaluations, we recommend you to export following environment variables to your environment. Some are necessary for certain tasks to run. + +```bash +export OPENAI_API_KEY="" +export HF_HOME="" +export HF_TOKEN="" +export HF_HUB_ENABLE_HF_TRANSFER="1" +export REKA_API_KEY="" +# Other possible environment variables include +# ANTHROPIC_API_KEY,DASHSCOPE_API_KEY etc. +``` + +## Some common environment issue +Sometimes you might encounter some common issues for example error related to `httpx` or `protobuf`. To solve these issues, you can first try + +```bash +python3 -m pip install httpx==0.23.3; +python3 -m pip install protobuf==3.20; +# If you are using numpy==2.x, sometimes may causing errors +python3 -m pip install numpy==1.26; +# Someties sentencepiece are required for tokenizer to work +python3 -m pip install sentencepiece; +``` + +# Image Model + +### LLaVA +First, you will need to clone repo of `lmms_eval` and repo of [`llava`](https://github.com/LLaVA-VL/LLaVA-NeXT/tree/inference) + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +cd /path/to/LLaVA-NeXT; +python3 -m pip install -e ".[train]"; + + +TASK=$1 +CKPT_PATH=$2 +CONV_TEMPLATE=$3 +MODEL_NAME=$4 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +#mmbench_en_dev,mathvista_testmini,llava_in_the_wild,mmvet +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model llava \ + --model_args pretrained=$CKPT_PATH,conv_template=$CONV_TEMPLATE,model_name=$MODEL_NAME \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ +``` +If you are trying to use large LLaVA models such as LLaVA-NeXT-Qwen1.5-72B, you can try adding `device_map=auto` in model_args and change `num_processes` to 1. + +### IDEFICS2 + +You won't need to clone any other repos to run idefics. Making sure your transformers version supports idefics2 would be enough + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +python3 -m pip install transformers --upgrade; + +TASK=$1 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model idefics2 \ + --model_args pretrained=HuggingFaceM4/idefics2-8b \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ +``` + +### InternVL2 + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + + +python3 -m pip install flash-attn --no-build-isolation; +python3 -m pip install torchvision einops timm sentencepiece; + + +TASK=$1 +CKPT_PATH=$2 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +#ai2d,chartqa,docvqa_val,mmmu,infovqa_val,mmbench_en_dev,mme,ok_vqa_val2014,pope,scienceqa_img,realworldqa,textvqa_val +accelerate launch --num_processes 8 --main_process_port 12380 -m lmms_eval \ + --model internvl2 \ + --model_args pretrained=$CKPT_PATH \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ +``` + + +### InternVL-1.5 +First you need to fork [`InternVL`](https://github.com/OpenGVLab/InternVL) + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +cd /path/to/InternVL/internvl_chat +python3 -m pip install -e .; + +python3 -m pip install flash-attn==2.3.6 --no-build-isolation; + + +TASK=$1 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model internvl \ + --model_args pretrained="OpenGVLab/InternVL-Chat-V1-5"\ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + +### Xcomposer-4KHD and Xcomposer-2d5 + +Both of these two models does not require external repo + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + + +python3 -m pip install flash-attn --no-build-isolation; +python3 -m pip install torchvision einops timm sentencepiece; + +TASK=$1 +MODALITY=$2 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +# For Xcomposer2d5 +accelerate launch --num_processes 8 --main_process_port 10000 -m lmms_eval \ + --model xcomposer2d5 \ + --model_args pretrained="internlm/internlm-xcomposer2d5-7b",device="cuda",modality=$MODALITY\ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +# For Xcomposer-4kHD +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model xcomposer2_4khd \ + --model_args pretrained="internlm/internlm-xcomposer2-4khd-7b" \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + +### InstructBLIP + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +python3 -m pip install transformers --upgrade; + +CKPT_PATH=$1 +TASK=$2 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model instructblip \ + --model_args pretrained=$CKPT_PATH \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix instructblip \ + --output_path ./logs/ + +``` + +### SRT API MODEL +To enable faster testing speed for larger llava model, you can use this srt api model to enable testing through sglang. + +Here are the scripts if you want to test the result in one script. +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +cd /path/to/sglang; +python3 -m pip install -e "python[all]"; + + +python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ + + +CKPT_PATH=$1 +TOK_PATH=$2 +TASK=$3 +MODALITY=$4 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +# Serve sglang in backend +python3 -m sglang.launch_server --model-path ${CKPT_PATH} --tokenizer-path ${TOK_PATH} --port=30000 --host=127.0.0.1 --tp-size=8 --chat-template=chatml-llava & + +# Wait till the server is ready +sleep 360; + +python3 -m lmms_eval \ + --model srt_api \ + --model_args modality=$MODALITY,host=127.0.0.1,port=30000,timeout=600 \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + +You can use the script in `sglang` under `test` folder to kill all sglang service + +# API Model + +### GPT + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +export OPENAI_API_KEY="" + +TASK=$1 +MODEL_VERSION=$2 +MODALITIES=$3 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 30000 -m lmms_eval \ + --model gpt4v \ + --model_args model_version=$MODEL_VERSION,modality=$MODALITIES\ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + + +### Claude + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +export ANTHROPIC_API_KEY="" + +TASK=$1 +MODEL_VERSION=$2 +MODALITIES=$3 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model claude \ + --model_args model_version=$MODEL_VERSION\ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ +``` + + +# Video Model + +### LLaVA-VID + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +cd /path/to/LLaVA-NeXT; +python3 -m pip install -e ".[train]"; + +python3 -m pip install flash-attn --no-build-isolation; + +python3 -m pip install av; + + +TASK=$1 +CKPT_PATH=$2 +CONV_TEMPLATE=$3 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model llavavid \ + --model_args pretrained=$CKPT_PATH,conv_template=$CONV_TEMPLATE,video_decode_backend=decord,max_frames_num=32 \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + + +### LLaMA-VID + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +# Notice that you should not leave the folder of LLaMA-VID when calling lmms-eval +# Because they left their processor's config inside the repo +cd /path/to/LLaMA-VID; +python3 -m pip install -e . + +python3 -m pip install av sentencepiece; + +TASK=$1 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model llama_vid \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ +``` + +### Video-LLaVA + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +python3 -m pip install transformers --upgrade; +python3 -m pip install av sentencepiece; + + +TASK=$1 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model video_llava \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + + +### MPlug-Owl +Notice that this model will takes long time to load, please be patient :) + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +# It has to use an old transformers version to run +python3 -m pip install av sentencepiece protobuf==3.20 transformers==4.28.1 einops; + +TASK=$1 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model mplug_owl_video \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ + +``` + + +### Video-ChatGPT + +```bash +cd /path/to/lmms-eval +python3 -m pip install -e .; + +python3 -m pip install sentencepiece av; + +TASK=$1 +echo $TASK +TASK_SUFFIX="${TASK//,/_}" +echo $TASK_SUFFIX + +accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \ + --model video_chatgpt \ + --tasks $TASK \ + --batch_size 1 \ + --log_samples \ + --log_samples_suffix $TASK_SUFFIX \ + --output_path ./logs/ +``` \ No newline at end of file From 07bdc08174f04fc68bb6ba6f0668cc4f928e4999 Mon Sep 17 00:00:00 2001 From: kcz358 Date: Thu, 1 Aug 2024 03:22:04 +0000 Subject: [PATCH 19/36] Delete some comments --- docs/run_examples.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/run_examples.md b/docs/run_examples.md index 91723b060..99c2e7162 100644 --- a/docs/run_examples.md +++ b/docs/run_examples.md @@ -102,7 +102,6 @@ echo $TASK TASK_SUFFIX="${TASK//,/_}" echo $TASK_SUFFIX -#ai2d,chartqa,docvqa_val,mmmu,infovqa_val,mmbench_en_dev,mme,ok_vqa_val2014,pope,scienceqa_img,realworldqa,textvqa_val accelerate launch --num_processes 8 --main_process_port 12380 -m lmms_eval \ --model internvl2 \ --model_args pretrained=$CKPT_PATH \ From 27535fcb132f2e3d5e73c00bf34ce1968ad65ee7 Mon Sep 17 00:00:00 2001 From: Pu Fanyi Date: Wed, 31 Jul 2024 22:05:42 -0700 Subject: [PATCH 20/36] LiveBench Rollback (#152) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task * Update lmms_eval configuration to use process_results_use_image and full_docs options * split tasks * chore: Update live_bench task dataset names and versions * rollback --- lmms_eval/tasks/live_bench/live_bench.yaml | 1 - tools/live_bench/example.ipynb | 481 ++++++++++++++++ tools/live_bench/live_bench/api/live_bench.py | 20 + .../data_generator/live_bench_data.py | 139 +++++ .../live_bench/data_generator/qa_generator.py | 522 ++++++++++++++++++ .../live_bench/data_generator/response.py | 12 + .../live_bench/data_generator/score_getter.py | 157 ++++++ .../live_bench/data_generator/utils/claude.py | 68 +++ .../live_bench/data_generator/utils/gemini.py | 37 ++ tools/live_bench/live_bench/driver/.gitignore | 1 + .../live_bench/driver/load_driver.py | 71 +++ .../live_bench/screen_shoter/screen.py | 30 + .../live_bench/screen_shoter/screen_shoter.py | 141 +++++ .../live_bench/websites/load_website.py | 34 ++ .../live_bench/live_bench/websites/website.py | 62 +++ .../live_bench/websites/website_list.yaml | 78 +++ tools/live_bench/pyproject.toml | 0 tools/live_bench/setup.py | 0 18 files changed, 1853 insertions(+), 1 deletion(-) create mode 100644 tools/live_bench/example.ipynb create mode 100644 tools/live_bench/live_bench/api/live_bench.py create mode 100644 tools/live_bench/live_bench/data_generator/live_bench_data.py create mode 100644 tools/live_bench/live_bench/data_generator/qa_generator.py create mode 100644 tools/live_bench/live_bench/data_generator/response.py create mode 100644 tools/live_bench/live_bench/data_generator/score_getter.py create mode 100644 tools/live_bench/live_bench/data_generator/utils/claude.py create mode 100644 tools/live_bench/live_bench/data_generator/utils/gemini.py create mode 100644 tools/live_bench/live_bench/driver/.gitignore create mode 100644 tools/live_bench/live_bench/driver/load_driver.py create mode 100644 tools/live_bench/live_bench/screen_shoter/screen.py create mode 100644 tools/live_bench/live_bench/screen_shoter/screen_shoter.py create mode 100644 tools/live_bench/live_bench/websites/load_website.py create mode 100644 tools/live_bench/live_bench/websites/website.py create mode 100644 tools/live_bench/live_bench/websites/website_list.yaml mode change 100755 => 100644 tools/live_bench/pyproject.toml mode change 100755 => 100644 tools/live_bench/setup.py diff --git a/lmms_eval/tasks/live_bench/live_bench.yaml b/lmms_eval/tasks/live_bench/live_bench.yaml index 1253105ee..901a12a87 100755 --- a/lmms_eval/tasks/live_bench/live_bench.yaml +++ b/lmms_eval/tasks/live_bench/live_bench.yaml @@ -2,7 +2,6 @@ group: live_bench task: - live_bench_2406 - live_bench_2407 - metadata: api_type : openai eval_with_mini: false diff --git a/tools/live_bench/example.ipynb b/tools/live_bench/example.ipynb new file mode 100644 index 000000000..92b9df73e --- /dev/null +++ b/tools/live_bench/example.ipynb @@ -0,0 +1,481 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/pufanyi/anaconda3/anacondabin/envs/live_bench/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from random import sample\n", + "\n", + "from live_bench.websites.website import DefaultWebsite\n", + "from live_bench.websites import load_websites\n", + "\n", + "# website = load_websites()\n", + "# website = sample(website, 1)\n", + "# website[0].url\n", + "website = [DefaultWebsite(url=\"https://www.asahi.com/\")] # , DefaultWebsite(url=\"https://www.bbc.com/sport\"), DefaultWebsite(url=\"https://www.bbc.com/business\")]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "could not detect version_main.therefore, we are assuming it is chrome 108 or higher\n" + ] + } + ], + "source": [ + "from live_bench.data_generator.utils.extract_infomation import InfomationExtractor\n", + "from live_bench.screen_shoter import get_shoter\n", + "from live_bench.driver import load_driver\n", + "\n", + "shoter = get_shoter(\"single_screen\")\n", + "driver = load_driver()\n", + "w = shoter(driver, website[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "extractor = InfomationExtractor()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "response = extractor.extract_infomation(w)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "**Here is something you can take as reference.**\n", + "\n", + "## Text Extracted in the HTML\n", + "\n", + "Below is the text extracted from the website for you to take reference:\n", + "BBC Home - Breaking News, World News, US News, Sports, Business, Innovation, Climate, Culture, Travel, Video & Audio\n", + "\n", + "What son's conviction means for President Biden\n", + "The guilty verdict is unlikely to change voters' minds, but it will be a personal blow for the US president.\n", + "1 hr ago | US & Canada\n", + "\n", + "Hunter Biden found guilty on all counts in gun trial\n", + "The US president's son is found guilty of lying about his drug use when buying a handgun in 2018.\n", + "- The struggles and scandals of Hunter Biden\n", + "\n", + "Blinken says fate of ceasefire plan down to Hamas\n", + "The US diplomat says Israel's prime minister \"reaffirmed his commitment\" to a Gaza ceasefire plan.\n", + "2 hrs ago | Middle East\n", + "\n", + "Ukraine 'hits missile launch sites in Russia'\n", + "The mayor of the city of Kharkiv says the situation there is \"calmer\" as Russia has been shelling less.\n", + "1 hr ago | Europe\n", + "\n", + "Four US college instructors stabbed in public park in China\n", + "The instructors were on a daytime visit to a public park when they were attacked, Cornell College says.\n", + "4 hrs ago | Asia\n", + "\n", + "Animal-rights protesters attack portrait of King\n", + "Animal-rights protesters attack a portrait of King Charles III, in a London gallery.\n", + "2 hrs ago | UK\n", + "\n", + "Warning shots from South as NK soldiers cross border\n", + "The incident at the DMZ comes at a time of heightened tensions between the two Koreas.\n", + "11 hrs ago | Asia\n", + "\n", + "## Image Features\n", + "\n", + "From the screenshot of the news website you provided, here is the analysis based on the images displayed alongside the corresponding news headings and text:\n", + "\n", + "1. **Image Associated with Hunter Biden's Conviction**:\n", + " - **Description**: The image depicts Hunter Biden escorted by, possibly, security personnel or aides. He seems to be dressed in a formal dark suit and appears to be descending stairs or possibly exiting a vehicle. He carries an air of seriousness, likely reflective of the gravity of his legal situation.\n", + " - **Relevance**: This image effectively captures the serious, personal, and public nature of the judicial proceedings against the President's son, making the situation more relatable to the audience. It directly ties to the news confirming Hunter Biden’s guilty verdict in a gun trial related to lying about drug use.\n", + "\n", + "2. **Image Accompanying the Article on Biden's Supporters**:\n", + " - **Description**: The accompanying image shows a group of enthusiastic supporters holding signs, with one prominently reading \"Say Yes to Biden,\" suggesting a political rally or campaign event. The participants display expressions of support and enthusiasm.\n", + " - **Relevance**: This image provides a visual contrast to the first, highlighting the ongoing support for the Biden family or campaign despite the legal issues faced by Hunter Biden. It serves to illustrate the political backdrop and public opinion dynamic mentioned in the news headlines.\n", + "\n", + "These images serve different purposes:\n", + "- The first image personalizes the news story, putting a face to the name in a high-stakes legal controversy. It underlines the personal and public challenges faced by the Biden family due to the conviction.\n", + "- The second image contextualizes the broader political support for the Biden family, suggesting that despite personal legal woes, there is a segment of the populace fervently supporting them.\n", + "\n", + "The clear connection between the images and the corresponding textual content on the news site helps readers visualize and better understand the unfolding events, enhancing the impact of the news storytelling.\n", + "\n", + "## Interesting Points\n", + "\n", + "The BBC news website, as demonstrated through the detailed examination of its content, offers a dynamic and visually engaging approach to news presentation. Here’s a deeper analysis of how it distinguishes itself:\n", + "\n", + "1. **Comprehensive and Geographically Diverse News Coverage**:\n", + " - The content spans a wide range of geographical locations including the US, Middle East, Europe, Asia, and the UK. Each news piece targets a major recent event, reflecting the website’s commitment to global news coverage. This expansive geographic focus ensures that readers have access to a broad spectrum of significant, impactful news.\n", + "\n", + "2. **Varied Content Themes**: \n", + " - The news themes are diverse, covering political, social, and cultural issues. From the legal troubles of a high-profile political figure’s son in the US to a ceasefire plan in the Middle East and violent incidents in Asia, the website covers a wide array of topics. This variety meets different readers' interests and keeps the content engaging.\n", + "\n", + "3. **Immediate Relevance**:\n", + " - The website's content is timely, as indicated by timestamps such as “1 hr ago” and “2 hrs ago.” This reflects the website’s commitment to providing the latest news, which is crucial for maintaining reader engagement and trust in a digital age where current information is highly valued.\n", + "\n", + "4. **Stylistic and Engaging Visual Design**:\n", + " - The use of compelling images alongside the news articles plays a critical role in storytelling. For instance, the image of Hunter Biden descending steps with a serious demeanor visually reinforces the gravity of the news about his conviction. \n", + " - Meanwhile, the image of supporters holding \"Say Yes to Biden\" signs juxtaposed with Hunter Biden's legal news offers a visual narrative of continued support amidst political strife, underscoring the complexity and depth of public and personal life in politics.\n", + "\n", + "5. **Interactive and Multimedia Features**:\n", + " - The use of tags such as \"OLIVE\" beside the breaking story of Hunter Biden indicates an interactive or breaking news feature that likely offers real-time updates and extensive coverage. This kind of multimedia integration enhances user interaction and engagement.\n", + "\n", + "In summary, the BBC news website sets itself apart through a combination of up-to-date, visually engaging, and comprehensively covered news items that cater to a global audience with varied interests. The effective use of images not only contextualizes the stories but also adds a layer of emotional and visual impact, making the news relatable and striking.\n" + ] + } + ], + "source": [ + "print(str(response))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This screenshot from a news website contains several images corresponding to different news stories. Let's examine each image and extract relevant details:\n", + "\n", + "1. **Image associated with the Michael Mosley news story:**\n", + " - The image depicts a middle-aged couple, smiling warmly at each other in a sunny, natural outdoor setting. This photo likely portrays Dr. Michael Mosley with his wife, representing a human interest angle to the story about Dr. Mosley's disappearance on the Greek island of Symi. The caption, \"'We will not lose hope,' says Michael Mosley's wife,\" implies a context of hope and determination amidst difficult circumstances.\n", + "\n", + "2. **Image linked to the news about the hostages freed in Gaza:**\n", + " - This image features a group of soldiers with one individual in civilian clothing at the center, being lifted or celebrated, possibly right after a rescue scenario. The setting appears to be a rugged outdoor area, suggestive of a conflict or military zone, which aligns with the news story about hostages being freed in Gaza. The inclusion of armed personnel and a jubilant expression on the civilian's face highlights the relief and successful outcome of a dangerous operation.\n", + "\n", + "3. **Image for the Nova festival hostages news:**\n", + " - This image depicts a motorboat on clear water under bright skies, possibly implying the geographic setting related to Michael Mosley’s disappearance near the Greek island of Symi. The serene environment contrasts starkly with the concerning news of his disappearance during what might have been a routine outing or travel.\n", + "\n", + "These images serve as visual supplements to the written content, providing readers with a clearer, more immediate understanding of the stories. They help bridge the emotional and contextual gaps that pure text might leave, allowing readers to engage more deeply with the news events. Each image is carefully selected to evoke specific sentiments and to provide visual context to the news headlines and summaries.\n" + ] + } + ], + "source": [ + "print(response[\"features\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from live_bench import LiveBench" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = LiveBench(force_clear=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'2024-06'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.name" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "could not detect version_main.therefore, we are assuming it is chrome 108 or higher\n", + "Capturing websites: 0%| | 0/1 [00:00 str: + subtask = subtask.strip().lower() + for valid_subtask in SUBTASKS: + if valid_subtask.lower() in subtask.lower(): + return valid_subtask + return "Unknown Subtask" + + def set_subtask(self, subtask: str): + """ + Set the subtask for the QAData instance after parsing it. + + Args: + subtask (str): The subtask string to be set. + """ + self.subtask = self.parse_subtask(subtask) + + def to_dict(self): + return {"question": self.question, "answer": self.answer} + + +class QAGenerator(ABC): + def __init__(self, prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md")): + self.prompt_file = prompt_file + self.prompt = self._load_prompt() + + def _load_prompt(self): + with open(self.prompt_file, "r") as f: + return f.read() + + def __call__(self, images: ScreenImage, *args, **kwargs): + return self.generate(images, *args, **kwargs) + + def generate(self, images: ScreenImage, *, test=False, infomation=None, **kwargs) -> Response: + if test: + return Response(success=True, content="This is a test response.", full_log={}) + return self._generate(images, infomation=infomation, test=test, **kwargs) + + def check(self, images: ScreenImage, question, answer, criteria, subtask, *, infomation=None, test=False, **kwargs) -> Response: + if test: + return Response(success=True, content="This is a test response.", full_log={}) + return self._check(images, question, answer, criteria, subtask, infomation=infomation, **kwargs) + + @abstractmethod + def _generate(self, images: ScreenImage, **kwargs) -> Response: + raise NotImplementedError("_generate not implemented") + + @abstractmethod + def _check(self, images: ScreenImage, question, answer, criteria, subtask, **kwargs) -> Response: + raise NotImplementedError("_check not implemented") + + def format_response(self, response: Response) -> QAData: + if response.success: + qa_data = self._format_response(response) + if qa_data is None: + return [] + else: + return qa_data + else: + return [] + + @abstractmethod + def _format_response(self, response: Response) -> str: + raise NotImplementedError("format_response not implemented") + + @abstractmethod + def format_checked_response(self, response: Response) -> QAData: + raise NotImplementedError("format_checked_response not implemented") + + def get_name(self) -> str: + raise NotImplementedError("get_name not implemented") + + +class GeneratorRegistry: + def __init__(self): + self.generators = {} + + def register_generator(self, name): + def decorator(cls): + self.generators[name] = cls + cls.get_name = lambda self: name + return cls + + return decorator + + def get_generator(self, name) -> QAGenerator: + return self.generators[name] + + def get_random_generator(self) -> QAGenerator: + return random.choice(list(self.generators.values())) + + +generator_registry = GeneratorRegistry() + + +def register_generator(name): + return generator_registry.register_generator(name) + + +def get_generator(name, *args, **kwargs) -> QAGenerator: + return generator_registry.get_generator(name)(*args, **kwargs) + + +def get_random_generator(*args, **kwargs) -> QAGenerator: + return generator_registry.get_random_generator()(*args, **kwargs) + + +@register_generator("gpt4v") +class GPT4Generator(QAGenerator): + def __init__( + self, + prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), + model="gpt-4o", + example_path=os.path.join(os.path.dirname(__file__), "example"), + check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), + ): + super().__init__(prompt_file) + API_KEY = os.getenv("OPENAI_API_KEY") + if not API_KEY: + raise ValueError("OPENAI_API_KEY environment variable not set.") + self.api_key = API_KEY + self.client = openai.OpenAI(api_key=self.api_key) + self.model = model + if os.path.exists(example_path): + self.example_path = example_path + else: + self.example_path = None + if os.path.exists(check_prompt): + with open(check_prompt, "r") as f: + self.check_prompt = f.read() + else: + self.check_prompt = check_prompt + + def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation): + example = [ + { + "type": "text", + "text": "Here are few examples about the task and the expected output format. You can take these as examples to generate your own questions.", + }, + format_gpt4v_images(example_image), + { + "type": "text", + "text": example_output, + }, + ] + content = example + [format_gpt4v_images(image) for image in images] + if infomation: + content.append({"type": "text", "text": str(infomation)}) + content.append( + { + "type": "text", + "text": "Please generate high-quality questions focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", + }, + ) + messages = [ + { + "role": "system", + "content": self.prompt, + }, + { + "role": "user", + "content": content, + }, + ] + return messages + + def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation=None, **kwargs): + if self.example_path: + example_image_path = os.path.join(self.example_path, "example_website.png") + example_output_path = os.path.join(self.example_path, "example_output.json") + example_image = Image.open(example_image_path) + with open(example_output_path, "r") as f: + example_output = json.load(f) + example_output = json.dumps(example_output, indent=4) + + messages = self.format_messages(images.images, example_image, example_output, infomation) + + return gpt4v_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) + + def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): + messages = [ + { + "role": "system", + "content": self.check_prompt, + } + ] + content = [] + for img in images: + content.append(format_gpt4v_images(img)) + content.append( + { + "type": "text", + "text": f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}\nSubtask: {subtask}", + }, + ) + if infomation: + content.append( + { + "type": "text", + "text": str(infomation), + }, + ) + content.append( + { + "type": "text", + "text": "Please rephrase or rewrite the high-quality question focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", + }, + ) + messages.append( + { + "role": "user", + "content": content, + } + ) + return messages + + def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, **kwargs): + messages = self.get_check_prompt(question, answer, criteria, subtask, images.images) + return gpt4v_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) + + def format_checked_response(self, response: Response): + data = json.loads(response.content) + question = data.get("question", None) + answer = data.get("answer", None) + criteria = data.get("criteria", None) + subtask = data.get("subtask", None) + return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) + + def _format_response(self, response: Response) -> List[QAData]: + try: + qa_data = [] + content = json.loads(response.content) + for subtask, messages in content.items(): + subtask = subtask.lower() + for message in messages: + message_lower = {k.lower(): v for k, v in message.items()} + try: + question = message_lower["question"] + answer = message_lower["answer"] + criteria = message_lower["criteria"] + qa_data.append(QAData(question=question, answer=answer, criteria=criteria, subtask=subtask)) + except KeyError as e: + logger.error(f"Failed to parse response: {message}") + logger.error(f"Error: {e}") + return qa_data + except Exception as e: + logger.error(f"Failed to format response: {e}") + return [] + + +@register_generator("gemini") +class GeminiGenerator(QAGenerator): + def __init__( + self, + prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), + model="gemini-1.5-pro-latest", + example_path=os.path.join(os.path.dirname(__file__), "example"), + check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), + ): + super().__init__(prompt_file) + GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") + if not GOOGLE_API_KEY: + raise ValueError("GOOGLE_API_KEY environment variable not set.") + genai.configure(api_key=GOOGLE_API_KEY) + + self.api_key = GOOGLE_API_KEY + self.model = model + self.client = genai.GenerativeModel(model) + if os.path.exists(example_path): + self.example_path = example_path + else: + self.example_path = None + if os.path.exists(check_prompt): + with open(check_prompt, "r") as f: + self.check_prompt = f.read() + else: + self.check_prompt = check_prompt + + def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation = None): + content = [self.prompt, "\n", "Example Image:", example_image, "\n", "Example Output:", example_output] + content.extend(images) + content.append(str(infomation)) + content.append("Please generate high-quality questions focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.") + return content + + def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation: ImageInfomation = None, **kwargs): + if self.example_path: + example_image_path = os.path.join(self.example_path, "example_website.png") + example_output_path = os.path.join(self.example_path, "example_output.json") + example_image = Image.open(example_image_path) + with open(example_output_path, "r") as f: + # example_output = f.read() + example_output = json.load(f) + example_output = json.dumps(example_output, indent=4) + + messages = self.format_messages(images.images, example_image, example_output, infomation) + + return gemini_generate_response(self.client, messages, max_tokens, max_try_times, **kwargs) + + def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): + content = [self.check_prompt] + images + content.append(f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}, Subtask: {subtask}") + content.append("Your response should be strictly in the below format:\n\nQuestion: \nAnswer: \nCriteria: \nSubtask: ") + if infomation: + content.append(str(infomation)) + return content + + def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, infomation: ImageInfomation = None, **kwargs): + messages = self.get_check_prompt(question, answer, criteria, subtask, images.images, infomation) + return gemini_generate_response(self.client, messages, max_tokens, max_try_times, **kwargs) + + def format_checked_response(self, response: Response): + # Extract the question, answer, and subtask from the normalized content + question_match = re.search(r"question:\s*(.*?)\nAnswer:", response.content, re.IGNORECASE | re.DOTALL) + answer_match = re.search(r"answer:\s*(.*?)\nCriteria", response.content, re.IGNORECASE | re.DOTALL) + criteria_match = re.search(r"criteria:\s*(.*?)\n(Subtask:|$)", response.content, re.IGNORECASE | re.DOTALL) + subtask_match = re.search(r"subtask:\s*(.*)", response.content, re.IGNORECASE) + + question = answer = subtask = None + + if question_match: + # Extract the matched groups + question = question_match.group(1).strip() + if answer_match: + answer = answer_match.group(1).strip() + if criteria_match: + criteria = criteria_match.group(1).strip() + if subtask_match: + subtask = subtask_match.group(1).strip() + + return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) + + def _format_response(self, response: Response) -> List[QAData]: + try: + qa_data = [] + content = json.loads(response.content) + for subtask, message in content.items(): + subtask = subtask.lower() + message_lower = {k.lower(): v for k, v in message.items()} + try: + question = message_lower["question"] + answer = message_lower["answer"] + qa_data.append(QAData(question=question, answer=answer, subtask=subtask)) + except KeyError as e: + logger.error(f"Failed to parse response: {message}") + logger.error(f"Error: {e}") + return qa_data + except Exception as e: + logger.error(f"Failed to format response: {e}") + return [] + + +@register_generator("claude") +class ClaudeGenerator(QAGenerator): + def __init__( + self, + prompt_file: str = os.path.join(os.path.dirname(__file__), "prompt.md"), + model="claude-3-5-sonnet-20240620", + example_path=os.path.join(os.path.dirname(__file__), "example"), + check_prompt=os.path.join(os.path.dirname(__file__), "check_prompt.md"), + ): + super().__init__(prompt_file) + API_KEY = os.getenv("ANTHROPIC_API_KEY") + if not API_KEY: + raise ValueError("ANTHROPIC_API_KEY environment variable not set.") + self.api_key = API_KEY + self.client = anthropic.Anthropic(api_key=self.api_key) + self.model = model + if os.path.exists(example_path): + self.example_path = example_path + else: + self.example_path = None + if os.path.exists(check_prompt): + with open(check_prompt, "r") as f: + self.check_prompt = f.read() + else: + self.check_prompt = check_prompt + + def format_messages(self, images: List[Image.Image], example_image: Image.Image, example_output: str, infomation: ImageInfomation): + example = [ + { + "type": "text", + "text": "Here are few examples about the task and the expected output format. You can take these as examples to generate your own questions.", + }, + format_claude_images(example_image), + { + "type": "text", + "text": example_output, + }, + ] + content = example + [format_claude_images(image) for image in images] + if infomation: + content.append({"type": "text", "text": str(infomation)}) + content.append( + { + "type": "text", + "text": "Please generate high-quality questions focusing on the information displayed within this webpage. Ensure your response adheres to the examples provided above and is structured in JSON format, incorporating regular expressions to validate the format.", + }, + ) + messages = [ + { + "role": "user", + "content": content, + }, + ] + return messages + + def _generate(self, images: ScreenImage, *, max_tokens=4096, max_try_times=5, infomation=None, **kwargs): + if self.example_path: + example_image_path = os.path.join(self.example_path, "example_website.png") + example_output_path = os.path.join(self.example_path, "example_output.json") + example_image = Image.open(example_image_path) + with open(example_output_path, "r") as f: + # example_output = f.read() + example_output = json.load(f) + example_output = json.dumps(example_output, indent=4) + + messages = self.format_messages(images.images, example_image, example_output, infomation) + + return claude_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, system=self.prompt, **kwargs) + + def get_check_prompt(self, question: str, answer: str, criteria, subtask, images: List[Image.Image], infomation: ImageInfomation = None): + messages = [ + { + "role": "system", + "content": self.check_prompt, + } + ] + content = [] + for img in images: + content.append(format_claude_images(img)) + content.append( + { + "type": "text", + "text": f"Question: {question}\nQuestioner's Answer: {answer}\nCriteria: {criteria}\nSubtask: {subtask}", + }, + ) + if infomation: + content.append( + { + "type": "text", + "text": str(infomation), + }, + ) + content.append( + { + "type": "text", + "text": "Please rephrase or rewrite the high-quality question focusing on the information displayed within this webpage. Your response should be in the format of the examples provided above and in JSON format.", + }, + ) + messages.append( + { + "role": "user", + "content": content, + } + ) + return messages + + def _check(self, images: ScreenImage, question, answer, criteria, subtask, *, max_tokens=4096, max_try_times=5, **kwargs): + messages = self.get_check_prompt(question, answer, criteria, subtask, images.images) + return claude_generate_response(client=self.client, model=self.model, messages=messages, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) + + def format_checked_response(self, response: Response): + data = json.loads(response.content) + question = data.get("question", None) + answer = data.get("answer", None) + criteria = data.get("criteria", None) + subtask = data.get("subtask", None) + return QAData(question=question, answer=answer, criteria=criteria, subtask=subtask) + + def _format_response(self, response: Response) -> List[QAData]: + try: + qa_data = [] + content = json.loads(response.content) + for subtask, messages in content.items(): + subtask = subtask.lower() + for message in messages: + message_lower = {k.lower(): v for k, v in message.items()} + try: + question = message_lower["question"] + answer = message_lower["answer"] + criteria = message_lower["criteria"] + qa_data.append(QAData(question=question, answer=answer, criteria=criteria, subtask=subtask)) + except KeyError as e: + logger.error(f"Failed to parse response: {message}") + logger.error(f"Error: {e}") + return qa_data + except Exception as e: + logger.error(f"Failed to format response: {e}") + return [] diff --git a/tools/live_bench/live_bench/data_generator/response.py b/tools/live_bench/live_bench/data_generator/response.py new file mode 100644 index 000000000..9eed882be --- /dev/null +++ b/tools/live_bench/live_bench/data_generator/response.py @@ -0,0 +1,12 @@ +class Response(object): + def __init__(self, success: bool, content: str, full_log: dict): + self.success = success + self.content = content + self.full_log = full_log + + def to_dict(self): + return { + "success": self.success, + "content": self.content, + "full_log": self.full_log, + } diff --git a/tools/live_bench/live_bench/data_generator/score_getter.py b/tools/live_bench/live_bench/data_generator/score_getter.py new file mode 100644 index 000000000..412989285 --- /dev/null +++ b/tools/live_bench/live_bench/data_generator/score_getter.py @@ -0,0 +1,157 @@ +import os +import json +import random +import openai +import anthropic +from abc import ABC, abstractmethod +from typing import List +from PIL import Image +from live_bench.screen_shoter import ScreenImage +from live_bench.data_generator.qa_generator import Response +from live_bench.data_generator.utils.gpt4v import format_gpt4v_images, gpt4v_generate_response +from live_bench.data_generator.utils.claude import format_claude_images, claude_generate_response + + +class Score(object): + def __init__(self, score: int, reason: str): + self.score = score + self.reason = reason + + +class ScoreGetter(ABC): + def get_name(self): + return self.name + + @abstractmethod + def get_score(self, question: str, answer: str, images: ScreenImage): + raise NotImplementedError("get_score not implemented") + + def __call__(self, question: str, answer: str, images: ScreenImage, **kwargs): + return self.get_score(question, answer, images, **kwargs) + + +class ScoreGetterRegistry: + def __init__(self): + self.score_getters = {} + + def register_score_getter(self, name): + def decorator(cls): + self.score_getters[name] = cls + cls.name = name + return cls + + return decorator + + def get_score_getter(self, name) -> ScoreGetter: + return self.score_getters[name] + + def get_random_score_getter(self) -> ScoreGetter: + return random.choice(list(self.score_getters.values())) + + +generator_registry = ScoreGetterRegistry() + + +def register_score_getter(name): + return generator_registry.register_score_getter(name) + + +def get_score_getter(name, *args, **kwargs) -> ScoreGetter: + return generator_registry.get_score_getter(name)(*args, **kwargs) + + +def get_random_score_getter(*args, **kwargs) -> ScoreGetter: + return generator_registry.get_random_score_getter()(*args, **kwargs) + + +@register_score_getter("gpt4v") +class GPT4VScoreGetter(ScoreGetter): + def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="gpt-4o", example_path=os.path.join(os.path.dirname(__file__), "example")): + super().__init__() + if os.path.exists(prompt): + with open(prompt, "r") as f: + self.prompt = f.read() + else: + self.prompt = prompt + API_KEY = os.getenv("OPENAI_API_KEY") + if not API_KEY: + raise ValueError("OPENAI_API_KEY environment variable not set.") + self.api_key = API_KEY + self.client = openai.OpenAI(api_key=self.api_key) + self.model = model + if os.path.exists(example_path) and os.path.isfile(os.path.join(example_path, "example_score_input.md")): + with open(example_path, "r") as f: + self.example = f.read() + else: + self.example = None + + def _format_prompt(self, question: str, answer: str, images: List[Image.Image]): + prompt = [{"role": "system", "content": self.prompt}] + messages = [] + for image in images: + messages.append(format_gpt4v_images(image)) + messages.append({"type": "text", "text": f"Question: {question}\nQuestioner's Answer: {answer}"}) + messages.append({"type": "text", "text": 'You should format you answer into json format like this: {"reason": "some reason", "score": 10}'}) + prompt.append({"role": "user", "content": messages}) + return prompt + + def get_score(self, question: str, answer: str, images: ScreenImage, *, max_tokens=4096, max_try_times=5, **kwargs) -> Score: + prompt = self._format_prompt(question, answer, images) + try: + response = gpt4v_generate_response(client=self.client, model=self.model, messages=prompt, max_tokens=max_tokens, max_try_times=max_try_times, json_format=True, **kwargs) + if response.success: + content = json.loads(response.content) + score = content.get("score", None) + reason = content.get("reason", None) + return Score(score=score, reason=reason) + else: + return Score(score=None, reason=response.content) + except Exception as e: + return Score(score=None, reason=str(e)) + + +@register_score_getter("claude") +class ClaudeScoreGetter(ScoreGetter): + def __init__(self, prompt: str = os.path.join(os.path.dirname(__file__), "score_prompt.md"), model="claude-3-5-sonnet-20240620", example_path=os.path.join(os.path.dirname(__file__), "example")): + super().__init__() + if os.path.exists(prompt): + with open(prompt, "r") as f: + self.prompt = f.read() + else: + self.prompt = prompt + API_KEY = os.getenv("ANTHROPIC_API_KEY") + if not API_KEY: + raise ValueError("ANTHROPIC_API_KEY environment variable not set.") + self.api_key = API_KEY + self.client = anthropic.Anthropic(api_key=self.api_key) + self.model = model + if os.path.exists(example_path) and os.path.isfile(os.path.join(example_path, "example_score_input.md")): + with open(example_path, "r") as f: + self.example = f.read() + else: + self.example = None + + def _format_prompt(self, question: str, answer: str, images: List[Image.Image]): + # prompt = [{"role": "system", "content": self.prompt}] + prompt = [] + messages = [] + for image in images: + messages.append(format_claude_images(image)) + messages.append({"type": "text", "text": f"Question: {question}\nQuestioner's Answer: {answer}"}) + messages.append({"type": "text", "text": 'You should format you answer into JSON format like this: { "reason": "some reason", "score": 10 }'}) + prompt.append({"role": "user", "content": messages}) + return prompt + + def get_score(self, question: str, answer: str, images: ScreenImage, *, max_tokens=4096, max_try_times=5, **kwargs) -> Score: + prompt = self._format_prompt(question, answer, images) + try: + response = claude_generate_response(client=self.client, model=self.model, messages=prompt, system=self.prompt, max_tokens=max_tokens, max_try_times=max_try_times, **kwargs) + if response.success: + content = json.loads(response.content) + score = content.get("score", None) + reason = content.get("reason", None) + return Score(score=score, reason=reason) + else: + return Score(score=None, reason=response.content) + except Exception as e: + return Score(score=None, reason=str(e)) diff --git a/tools/live_bench/live_bench/data_generator/utils/claude.py b/tools/live_bench/live_bench/data_generator/utils/claude.py new file mode 100644 index 000000000..ccd0b381f --- /dev/null +++ b/tools/live_bench/live_bench/data_generator/utils/claude.py @@ -0,0 +1,68 @@ +from PIL import Image +import io +import base64 +from live_bench.data_generator.response import Response +import anthropic +import logging +from time import sleep +from typing import Union, List + +logger = logging.getLogger("lmms-eval") + + +def format_claude_images(image: Union[Image.Image, List[Image.Image]]): + if isinstance(image, list): + return [format_claude_images(img) for img in image] + buffered = io.BytesIO() + image.save(buffered, format="PNG") + img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") + return { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": img_str, + }, + } + + +def claude_generate_response(client: anthropic.Anthropic, model, messages, max_tokens: int = 4096, max_try_times=5, system=None, json_format="auto", test=False, **kwargs): + if json_format == "auto": + json_format = False + for message in messages: + if message.get("role") == "user": + contents = message.get("content", []) + if isinstance(contents, str): + if "json" in contents: + json_format = True + break + else: + for content in contents: + if content.get("type", None) == "text" and "json" in content.get("text", ""): + json_format = True + break + + if json_format: + messages.append({"role": "assistant", "content": "{"}) + + def _generate(): + if system: + return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, system=system, **kwargs) + else: + return client.messages.create(model=model, messages=messages, max_tokens=max_tokens, **kwargs) + + for times in range(max_try_times): + try: + response = _generate() + response_str = response.content[0].text + if json_format: + response_str = "{" + response_str + return Response(success=True, content=response_str, full_log={"input": messages, "output": response.to_dict()}) + except Exception as e: + logger.error(f"Failed to generate response: {e}") + if times < max_try_times - 1: + logger.info(f"Retrying... ({times+1}/{max_try_times})") + sleep(3) + else: + logger.error("Failed to generate response after retrying.") + return Response(success=False, content=str(e), full_log={"input": messages, "output": None}) diff --git a/tools/live_bench/live_bench/data_generator/utils/gemini.py b/tools/live_bench/live_bench/data_generator/utils/gemini.py new file mode 100644 index 000000000..57b77536c --- /dev/null +++ b/tools/live_bench/live_bench/data_generator/utils/gemini.py @@ -0,0 +1,37 @@ +import google.generativeai as genai +from time import sleep +from live_bench.data_generator.response import Response +import logging +from google.generativeai.types import HarmCategory, HarmBlockThreshold + +logger = logging.getLogger("lmms-eval") + + +def gemini_generate_response(client: genai.GenerativeModel, messages, max_tokens: int, max_try_times: int = 5, **kwargs): + generation_config = genai.GenerationConfig(max_output_tokens=max_tokens) + + def _generate(): + return client.generate_content( + messages, + generation_config=generation_config, + safety_settings={ + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + }, + **kwargs, + ) + + for times in range(max_try_times): + try: + response = _generate() + return Response(success=True, content=response.text, full_log={"input": messages, "output": response}) + except Exception as e: + logger.error(f"Failed to generate response: {e}") + if times < max_try_times - 1: + logger.info(f"Retrying... ({times+1}/{max_try_times})") + sleep(3) + else: + logger.error("Failed to generate response after retrying.") + return Response(success=False, content=str(e), full_log={"input": messages, "output": None}) diff --git a/tools/live_bench/live_bench/driver/.gitignore b/tools/live_bench/live_bench/driver/.gitignore new file mode 100644 index 000000000..0ef18421a --- /dev/null +++ b/tools/live_bench/live_bench/driver/.gitignore @@ -0,0 +1 @@ +extensions/ diff --git a/tools/live_bench/live_bench/driver/load_driver.py b/tools/live_bench/live_bench/driver/load_driver.py new file mode 100644 index 000000000..c0ca7528d --- /dev/null +++ b/tools/live_bench/live_bench/driver/load_driver.py @@ -0,0 +1,71 @@ +import os +import zipfile +import requests +from selenium import webdriver +from webdriver_manager.chrome import ChromeDriverManager +from webdriver_manager.core.os_manager import ChromeType +from selenium.webdriver.chrome.options import Options + +import undetected_chromedriver as uc + + +def load_driver( + window_size="auto", + headless=True, + driver="undetected_chromedriver", + driver_version=None, + chrome_type="CHROME", + adblock=True, + adblock_version="6.0.2-mv3", + extension_cache_dir=os.path.join(os.path.dirname(__file__), "extensions"), + *, + service=None, + additional_options=None, +): + options = Options() + if service is None: + chrome_type = chrome_type.upper() + if chrome_type == "CHROMIUM": + chrome_type = ChromeType.CHROMIUM + elif chrome_type == "CHROME": + chrome_type = ChromeType.GOOGLE + elif chrome_type == "BRAVE": + chrome_type = ChromeType.BRAVE + service = ChromeDriverManager(driver_version=driver_version, chrome_type=chrome_type).install() + if headless: + options.add_argument("--headless") + if adblock: + try: + adblock_url = f"https://code.getadblock.com/releases/adblockchrome-{adblock_version}.zip" + adblock_path = os.path.join(extension_cache_dir, f"adblockchrome-{adblock_version}") + if not os.path.isdir(adblock_path): + os.makedirs(os.path.join(adblock_path, ".."), exist_ok=True) + # Download the adblock zip file + response = requests.get(adblock_url) + with open(f"{adblock_path}.zip", "wb") as file: + file.write(response.content) + # Unzip the downloaded file + with zipfile.ZipFile(f"{adblock_path}.zip", "r") as zip_ref: + zip_ref.extractall(adblock_path) + # Remove the zip file after extraction + os.remove(f"{adblock_path}.zip") + options.add_argument(f"--load-extension={os.path.abspath(adblock_path)}") + except Exception as e: + print(f"Error loading adblock extension: {e}") + if driver == "undetected_chromedriver": + driver = uc.Chrome(headless=headless, options=options, driver_executable_path=service) + if window_size != "auto": + driver.set_window_size(*window_size) + return driver + elif driver == "chrome": + options = Options() + if additional_options is not None: + for option in additional_options: + options.add_argument(option) + service = webdriver.chrome.service.Service(service) + driver = webdriver.Chrome(service=service, options=options) + if window_size != "auto": + driver.set_window_size(*window_size) + return driver + else: + raise ValueError(f"Unknown driver: {driver}") diff --git a/tools/live_bench/live_bench/screen_shoter/screen.py b/tools/live_bench/live_bench/screen_shoter/screen.py new file mode 100644 index 000000000..5817b8cc6 --- /dev/null +++ b/tools/live_bench/live_bench/screen_shoter/screen.py @@ -0,0 +1,30 @@ +import io +import base64 + +from PIL import Image +from typing import List, Tuple + +from live_bench.websites import Website + + +def image_to_base64(image: Image.Image) -> str: + buffered = io.BytesIO() + image.save(buffered, format="PNG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + + +class ScreenImage(object): + def __init__(self, images: List[Image.Image], website: Website, shoter: str, screen_size: Tuple[int, int], capture_datetime: str): + self.images = images + self.website = website + self.shoter = shoter + self.screen_size = screen_size + self.capture_datetime = capture_datetime + + def to_dict(self): + return {"images": self.images, "website": self.website.get_info(), "shoter": self.shoter, "screen_size": self.screen_size, "capture_datetime": self.capture_datetime} + + def to_output_dict(self): + output = self.to_dict() + output["images"] = [image_to_base64(image) for image in self.images] + return output diff --git a/tools/live_bench/live_bench/screen_shoter/screen_shoter.py b/tools/live_bench/live_bench/screen_shoter/screen_shoter.py new file mode 100644 index 000000000..6069cd26d --- /dev/null +++ b/tools/live_bench/live_bench/screen_shoter/screen_shoter.py @@ -0,0 +1,141 @@ +from selenium import webdriver +from PIL import Image +from live_bench.websites import Website +from live_bench.screen_shoter.screen import ScreenImage +from typing import List +from abc import ABC, abstractmethod +from datetime import datetime +from PIL import Image +import os +import io +import logging + +logger = logging.getLogger("lmms-eval") + + +class ScreenShoter(ABC): + def __init__(self, screen_size=(1024, 1024)): + self.screen_size = screen_size + + def capture(self, driver: webdriver.Chrome, website: Website) -> ScreenImage: + if driver is not None: + website.visit(driver) + if self.screen_size != "auto": + driver.set_window_size(self.screen_size[0], self.screen_size[1]) + else: + driver.set_window_size(1024, 1024) + page_width = driver.execute_script("return document.body.scrollWidth") + driver.set_window_size(page_width, 1024) + # print("Screen size:", driver.get_window_size()) + images = self.get_screenshot(driver) + return ScreenImage(images, website, self.get_name(), self.screen_size, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + def __call__(self, driver: webdriver.Chrome, website: Website) -> List[Image.Image]: + return self.capture(driver, website) + + def get_name(self) -> str: + raise NotImplementedError("get_name not implemented") + + @abstractmethod + def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: + pass + + +class ScreenShoterRegistry: + def __init__(self): + self.shoters = {} + + def register_shoter(self, name): + def decorator(cls): + self.shoters[name] = cls + cls.get_name = lambda self: name + return cls + + return decorator + + def get_shoter(self, name) -> ScreenShoter: + return self.shoters[name] + + +shoter_registry = ScreenShoterRegistry() + + +def register_shoter(name): + return shoter_registry.register_shoter(name) + + +def get_shoter(name, *args, **kwargs) -> ScreenShoter: + return shoter_registry.get_shoter(name)(*args, **kwargs) + + +@register_shoter("human") +class HumanScreenShoter(ScreenShoter): + def __init__(self, screen_size=None): + super().__init__(screen_size) + + def capture(self, driver: webdriver.Chrome, website: Website) -> ScreenImage: + path = website.get_path() + images = [] + + def get_image(path): + try: + with open(path, "rb") as f: + image_data = f.read() + image = Image.open(io.BytesIO(image_data)) + images.append(image) + except Exception as e: + logger.error(f"Error loading image {path}: {e}") + + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + for file_name in files: + get_image(os.path.join(root, file_name)) + else: + try: + get_image(path) + except Exception as e: + logger.error(f"Error loading image {path}: {e}") + if not images: + raise ValueError(f"No images found in {path}") + return ScreenImage(images, website, self.get_name(), self.screen_size, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: + return [] + + +@register_shoter("single_screen") +class SingleScreenShoter(ScreenShoter): + def __init__(self, screen_size=(1024, 1024)): + super().__init__(screen_size) + + def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: + screenshot = driver.get_screenshot_as_png() + return [Image.open(io.BytesIO(screenshot))] + + +@register_shoter("rolling_screen") +class RollingScreenShoter(ScreenShoter): + def __init__(self, screen_size=(1024, 1024)): + super().__init__(screen_size) + + def get_screenshot(self, driver: webdriver.Chrome) -> List[Image.Image]: + screenshots = [] + # Scroll to the top of the page before taking the first screenshot + driver.execute_script("window.scrollTo(0, 0)") + # Get the total height of the web page + total_height = driver.execute_script("return document.body.parentNode.scrollHeight") + # Get the viewport height + viewport_height = driver.execute_script("return window.innerHeight") + # Initialize the current scroll position + current_scroll_position = 0 + + # Scroll through the page and take screenshots + while current_scroll_position < total_height: + # Take screenshot and append to the list + screenshot = driver.get_screenshot_as_png() + screenshots.append(Image.open(io.BytesIO(screenshot))) + # Scroll down by the viewport height + current_scroll_position += viewport_height + driver.execute_script(f"window.scrollTo(0, {current_scroll_position})") + + return screenshots diff --git a/tools/live_bench/live_bench/websites/load_website.py b/tools/live_bench/live_bench/websites/load_website.py new file mode 100644 index 000000000..10976db75 --- /dev/null +++ b/tools/live_bench/live_bench/websites/load_website.py @@ -0,0 +1,34 @@ +import yaml +import os +from random import sample +from live_bench.websites.website import Website, DefaultWebsite, HumanScreenShotWebsite + + +def get_website(website_dict): + if "website_class" not in website_dict: + website_class = DefaultWebsite + else: + website_class = website_dict["website_class"] + url = website_dict["url"] + if "args" in website_dict: + return website_class(url, **website_dict["args"]) + else: + return website_class(url) + + +def load_websites(num_sample: int = -1): + website_list_path = os.path.join(os.path.dirname(__file__), "website_list.yaml") + with open(website_list_path, "r") as f: + website_list = yaml.full_load(f)["websites"] + if num_sample > 0: + website_list = sample(website_list, num_sample) + return [get_website(website_dict) for website_dict in website_list] + + +def load_websites_from_file(file_path): + names = os.listdir(file_path) + websites = [] + for name in names: + path = os.path.join(file_path, name) + websites.append(HumanScreenShotWebsite(path=path, name=name)) + return websites diff --git a/tools/live_bench/live_bench/websites/website.py b/tools/live_bench/live_bench/websites/website.py new file mode 100644 index 000000000..327bad372 --- /dev/null +++ b/tools/live_bench/live_bench/websites/website.py @@ -0,0 +1,62 @@ +import time +import os + +from webdriver_manager.core.driver import Driver +from abc import ABC, abstractmethod + + +class Website(ABC): + def __init__(self, url=None, name=None, path=None): + self.url = url + self.name = name + self.path = path + assert self.url is not None or self.path is not None, "Either url or path must be provided" + + def get_path(self): + if self.url: + return self.url + else: + return self.path + + def visit(self, driver: Driver): + self.pre_visit(driver) + driver.get(self.url) + self.post_visit(driver) + + def get_info(self): + info = {} + if self.url: + info["url"] = self.url + if self.name: + info["name"] = self.name + return info + + @abstractmethod + def pre_visit(self, driver: Driver): + raise NotImplementedError("pre_action not implemented") + + @abstractmethod + def post_visit(self, driver: Driver): + raise NotImplementedError("post_action not implemented") + + +class DefaultWebsite(Website): + def __init__(self, url, name=None): + super().__init__(url, name) + + def pre_visit(self, driver: Driver): + pass + + def post_visit(self, driver: Driver): + time.sleep(5) # Wait for 5 seconds to allow adblock to finish + + +class HumanScreenShotWebsite(Website): + def __init__(self, name=None, path=None): + super().__init__(name=name, path=path) + + def pre_visit(self, driver: Driver): + pass + + def post_visit(self, driver: Driver): + pass diff --git a/tools/live_bench/live_bench/websites/website_list.yaml b/tools/live_bench/live_bench/websites/website_list.yaml new file mode 100644 index 000000000..c85605d7d --- /dev/null +++ b/tools/live_bench/live_bench/websites/website_list.yaml @@ -0,0 +1,78 @@ +websites: +- url: https://www.bbc.com/ + # can add below line to specify the class to use for this website + # website_class: !constructor website.DefaultWebsite + # can add args tag to specify the arguments to pass to the class constructor + # args: + # arg1: value1 + # arg2: value2 +# - url: https://www.bbc.com/news +# - url: https://www.bbc.com/sport +# - url: https://www.bbc.com/business +# - url: https://www.bbc.com/innovation +# - url: https://www.bbc.com/culture +# - url: https://www.bbc.com/travel +# - url: https://www.bbc.com/future-planet +# - url: https://edition.cnn.com/ +# - url: https://edition.cnn.com/politics +# - url: https://edition.cnn.com/entertainment +# - url: https://edition.cnn.com/style +# - url: https://www.bloomberg.com/economics +# - url: https://www.bloomberg.com/industries +# - url: https://www.bloomberg.com/technology +# - url: https://www.bloomberg.com/politics +# - url: https://www.bloomberg.com/opinion +# - url: https://www.wsj.com/ +# - url: https://www.wsj.com/world/africa?mod=nav_top_subsection +# - url: https://www.wsj.com/world/americas?mod=nav_top_subsection +# - url: https://www.wsj.com/world/asia?mod=nav_top_subsection +# - url: https://www.wsj.com/world/china?mod=nav_top_subsection +# - url: https://www.wsj.com/world/europe?mod=nav_top_subsection +# - url: https://www.wsj.com/world/middle-east?mod=nav_top_subsection +# - url: https://www.wsj.com/world/india?mod=nav_top_subsection +# - url: https://www.wsj.com/world/oceania?mod=nav_top_subsection +# - url: https://www.wsj.com/world/russia?mod=nav_top_subsection +# - url: https://www.wsj.com/world/uk?mod=nav_top_subsection +# - url: https://www.wsj.com/science?mod=nav_top_subsection +# - url: https://www.wsj.com/science/archaeology?mod=nav_top_subsection +# - url: https://www.wsj.com/science/biology?mod=nav_top_subsection +# - url: https://www.wsj.com/science/environment?mod=nav_top_subsection +# - url: https://www.wsj.com/science/physics?mod=nav_top_subsection +# - url: https://www.wsj.com/science/space-astronomy?mod=nav_top_subsection +# - url: https://www.wsj.com/economy/central-banking?mod=nav_top_subsection +# - url: https://www.wsj.com/economy/consumers?mod=nav_top_subsection +# - url: https://www.wsj.com/economy/housing?mod=nav_top_subsection +# - url: https://www.wsj.com/economy/jobs?mod=nav_top_subsection +# - url: https://www.wsj.com/economy/trade?mod=nav_top_subsection +# - url: https://www.wsj.com/economy/global +# - url: https://www.wsj.com/tech/ai?mod=nav_top_subsection +# - url: https://www.wsj.com/tech/biotech +# - url: https://www.wsj.com/tech/cybersecurity?mod=nav_top_subsection +# - url: https://www.wsj.com/tech/personal-tech?mod=nav_top_subsection +# - url: https://www.reuters.com/ +# - url: https://www.reuters.com/business/aerospace-defense/ +# - url: https://www.reuters.com/business/autos-transportation/ +# - url: https://www.reuters.com/business/davos/ +# - url: https://www.reuters.com/business/energy/ +# - url: https://www.reuters.com/business/environment/ +# - url: https://www.reuters.com/business/finance/ +# - url: https://www.reuters.com/business/healthcare-pharmaceuticals/ +# - url: https://www.reuters.com/business/media-telecom/ +# - url: https://www.reuters.com/business/retail-consumer/ +# - url: https://www.reuters.com/business/future-of-health/ +# - url: https://www.reuters.com/business/future-of-money/ +# - url: https://www.reuters.com/business/take-five/ +# - url: https://www.reuters.com/business/world-at-work/ +# - url: https://www.reuters.com/breakingviews/ +# - url: https://www.reuters.com/technology/ +# - url: https://www.reuters.com/technology/cybersecurity/ +# - url: https://www.reuters.com/technology/space/ +# - url: https://www.reuters.com/technology/disrupted/ +# - url: https://www.reuters.com/technology/reuters-momentum/ +# - url: https://www.reuters.com/investigations/ +# - url: https://a16z.com/news-content/#latest +# - url: https://news.ycombinator.com/ +# - url: https://www.reddit.com/?rdt=48006 +# - url: https://news.crunchbase.com/ +# - url: https://www.cctv.com/ +# - url: https://sports.cctv.com/ diff --git a/tools/live_bench/pyproject.toml b/tools/live_bench/pyproject.toml old mode 100755 new mode 100644 diff --git a/tools/live_bench/setup.py b/tools/live_bench/setup.py old mode 100755 new mode 100644 From 2f636e7bf671f5cc4c12ccc07da341b9f8a520a4 Mon Sep 17 00:00:00 2001 From: Li Bo Date: Thu, 1 Aug 2024 15:06:59 +1000 Subject: [PATCH 21/36] add mvbench (#151) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task * Update lmms_eval configuration to use process_results_use_image and full_docs options * split tasks * chore: Update live_bench task dataset names and versions * feat: Add multi-choice parsing and processing functions This commit adds two new functions to the `egoschema/utils.py` file: `get_multi_choice_info` and `parse_multi_choice_response`. These functions are used to parse and process multi-choice responses in the Egoschema task. The `get_multi_choice_info` function extracts information about the available choices from the input document, while the `parse_multi_choice_response` function parses the generated response and returns the predicted index. These functions are essential for accurately processing multi-choice answers in the Egoschema task. * feat: Add regex-based parsing for multi-choice predictions This commit enhances the `perceptiontest_val_process_results_mc` function in the `utils.py` file. It introduces regex-based parsing to extract the predicted choice from the raw text prediction. If a match is found for A, B, C, or D, the matched letter is used as the prediction. Otherwise, an empty string is set as the prediction. This improvement ensures accurate processing of multi-choice predictions in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Improve accuracy calculation in perception test validation This commit refactors the `perceptiontest_val_aggregate_accuracy` function in the `utils.py` file. Instead of comparing the string representations of `answer_id` and `pred_id`, it now directly checks the `correct` field in the `accuracy` dictionary. This change ensures more accurate calculation of the overall accuracy in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * Refactor accuracy calculation in perception test validation * feat: Add SRT_API model to available models This commit adds the SRT_API model to the list of available models in the `__init__.py` file. This model can now be used for evaluation and testing purposes. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * chore: Update live_bench task dataset names and versions (cherry picked from commit 46a85b40b013503e52d007c96ca0607bd4604a3e) * refactor: Update question_for_eval key in MathVerseEvaluator This commit updates the key "question_for_eval" to "question" in the MathVerseEvaluator class. This change ensures consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update generate_submission_file function in mathverse utils This commit updates the `generate_submission_file` function in the `mathverse/utils.py` file. It adds the `problem_version` to the filename to ensure that the submission files are saved with the correct problem version. This change improves the organization and clarity of the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files This commit updates the default template YAML files in the `lmms_eval/tasks` directory. It modifies the `generation_kwargs` and `metadata` sections to improve the configuration and versioning of the tasks. These changes ensure consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Add llava_onevision model to seedbench_2.yaml This commit adds the `llava_onevision` model to the `seedbench_2.yaml` file in the `lmms_eval/tasks/seedbench_2` directory. The `llava_onevision` model now has the same `img_token` and `post_prompt` properties as the other models in the file. This change ensures consistency and improves the configuration of the task. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files * refactor: Update default template YAML files * refactor: Add lazy loading for images in mvbench_episodic_reasoning.yaml This commit adds lazy loading for images in the `mvbench_episodic_reasoning.yaml` file. The `doc_to_visual` function has been updated to include lazy loading functionality, improving the performance of the page load. This change ensures faster loading times for images in the `episodic_reasoning` dataset. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata This commit updates the Task class in the lmms_eval/api/task.py file to include lazy loading for metadata. The per_task_metadata dictionary is now updated with the config.metadata only if it is not empty. This change ensures that the metadata is properly updated and avoids unnecessary updates when the config.metadata is empty. Co-authored-by: [Co-author Name] <[coauthor@example.com]> --------- Co-authored-by: Fanyi Pu Co-authored-by: [Co-author Name] <[coauthor@example.com]> Co-authored-by: kcz358 --- lmms_eval/api/instance.py | 2 +- lmms_eval/api/task.py | 51 +++-- lmms_eval/models/llava_onevision.py | 7 + .../activitynetqa/_default_template_yaml | 2 +- lmms_eval/tasks/activitynetqa/utils.py | 14 +- lmms_eval/tasks/ai2d/ai2d.yaml | 2 +- lmms_eval/tasks/ai2d/utils.py | 14 +- lmms_eval/tasks/chartqa/chartqa.yaml | 2 +- lmms_eval/tasks/chartqa/utils.py | 6 +- lmms_eval/tasks/cvrr/_default_template_yaml | 2 +- lmms_eval/tasks/cvrr/utils.py | 14 +- .../_default_template_detailcaps_yaml | 2 +- lmms_eval/tasks/detailcaps/utils.py | 4 +- .../docvqa/_default_template_docvqa_yaml | 2 +- lmms_eval/tasks/docvqa/utils.py | 6 +- .../tasks/egoschema/_default_template_yaml | 2 +- lmms_eval/tasks/egoschema/utils.py | 14 +- lmms_eval/tasks/ferret/ferret.yaml | 2 +- lmms_eval/tasks/ferret/utils.py | 10 +- lmms_eval/tasks/gqa/gqa.yaml | 2 +- lmms_eval/tasks/gqa/utils.py | 6 +- lmms_eval/tasks/gqa_ru/gqa_ru.yaml | 2 +- lmms_eval/tasks/gqa_ru/utils.py | 6 +- .../tasks/hallusion_bench/evaluate_hb.py | 10 +- .../hallusion_bench_image.yaml | 2 +- .../iconqa/_default_template_docvqa_yaml | 2 +- lmms_eval/tasks/iconqa/utils.py | 8 +- lmms_eval/tasks/ii_bench/ii_bench.yaml | 2 +- lmms_eval/tasks/ii_bench/utils.py | 6 +- .../infovqa/_default_template_infovqa_yaml | 2 +- lmms_eval/tasks/infovqa/utils.py | 6 +- .../_default_template_internal_eval_yaml | 2 +- lmms_eval/tasks/internal_eval/utils.py | 8 +- lmms_eval/tasks/live_bench/live_bench.yaml | 1 + .../tasks/live_bench/live_bench_template_yaml | 3 +- lmms_eval/tasks/live_bench/utils.py | 10 +- .../llava-bench-coco/llava-bench-coco.yaml | 2 +- lmms_eval/tasks/llava-bench-coco/utils.py | 10 +- .../llava-in-the-wild/llava-in-the-wild.yaml | 2 +- lmms_eval/tasks/llava-in-the-wild/utils.py | 10 +- .../llava_interleave_bench/in_domain.yaml | 2 +- .../multi_view_in_domain.yaml | 2 +- .../llava_interleave_bench/out_of_domain.yaml | 2 +- .../tasks/llava_interleave_bench/utils.py | 26 +-- .../llava_wilder/llava_wilder_small.yaml | 2 +- lmms_eval/tasks/llava_wilder/utils.py | 10 +- .../longvideobench/longvideobench_val_i.yaml | 2 +- .../longvideobench/longvideobench_val_v.yaml | 2 +- lmms_eval/tasks/longvideobench/utils.py | 8 +- .../tasks/mathverse/mathverse_testmini.yaml | 2 +- .../mathverse_testmini_text_dominant.yaml | 2 +- .../mathverse_testmini_text_lite.yaml | 2 +- .../mathverse_testmini_text_only.yaml | 2 +- .../mathverse_testmini_vision_dominant.yaml | 2 +- .../mathverse_testmini_vision_intensive.yaml | 2 +- .../mathverse_testmini_vision_only.yaml | 2 +- lmms_eval/tasks/mathverse/utils.py | 6 +- lmms_eval/tasks/mathvista/mathvista_test.yaml | 2 +- .../mathvista/mathvista_testmini_cot.yaml | 2 +- .../mathvista/mathvista_testmini_format.yaml | 2 +- .../mathvista_testmini_solution.yaml | 2 +- lmms_eval/tasks/mathvista/utils.py | 10 +- lmms_eval/tasks/mlvu/utils.py | 2 +- .../mmbench/_default_template_mmbench_cn_yaml | 2 +- .../mmbench/_default_template_mmbench_en_yaml | 2 +- .../mmbench/_default_template_mmbench_ru_yaml | 2 +- lmms_eval/tasks/mmbench/cc_utils.py | 6 +- lmms_eval/tasks/mmbench/cn_utils.py | 6 +- lmms_eval/tasks/mmbench/en_utils.py | 6 +- lmms_eval/tasks/mmbench/mmbench_cc.yaml | 2 +- lmms_eval/tasks/mmbench/ru_utils.py | 6 +- lmms_eval/tasks/mme/mme.yaml | 2 +- lmms_eval/tasks/mme/utils.py | 10 +- lmms_eval/tasks/mmstar/mmstar.yaml | 2 +- lmms_eval/tasks/mmstar/utils.py | 10 +- lmms_eval/tasks/mmupd/mmaad_base.yaml | 2 +- lmms_eval/tasks/mmupd/mmaad_instruction.yaml | 2 +- lmms_eval/tasks/mmupd/mmaad_option.yaml | 2 +- lmms_eval/tasks/mmupd/mmiasd_base.yaml | 2 +- lmms_eval/tasks/mmupd/mmiasd_instruction.yaml | 2 +- lmms_eval/tasks/mmupd/mmiasd_option.yaml | 2 +- lmms_eval/tasks/mmupd/mmivqd_base.yaml | 2 +- lmms_eval/tasks/mmupd/mmivqd_instruction.yaml | 2 +- lmms_eval/tasks/mmupd/mmivqd_option.yaml | 2 +- lmms_eval/tasks/mmupd/utils.py | 6 +- lmms_eval/tasks/mmvet/mmvet.yaml | 2 +- lmms_eval/tasks/mmvet/utils.py | 8 +- lmms_eval/tasks/muirbench/muirbench.yaml | 2 +- lmms_eval/tasks/muirbench/utils.py | 6 +- .../tasks/multidocvqa/multidocvqa_test.yaml | 2 +- .../tasks/multidocvqa/multidocvqa_val.yaml | 2 +- lmms_eval/tasks/multidocvqa/utils.py | 6 +- .../_default_template.yaml | 2 +- .../utils.py | 10 +- .../tasks/mvbench/_default_template.yaml | 23 +++ lmms_eval/tasks/mvbench/mvbench.yaml | 24 +++ .../tasks/mvbench/mvbench_action_antonym.yaml | 8 + .../tasks/mvbench/mvbench_action_count.yaml | 8 + .../mvbench/mvbench_action_localization.yaml | 8 + .../mvbench/mvbench_action_prediction.yaml | 8 + .../mvbench/mvbench_action_sequence.yaml | 8 + .../mvbench/mvbench_character_order.yaml | 8 + .../mvbench_counterfactual_inference.yaml | 8 + .../mvbench_egocentric_navigation.yaml | 8 + .../mvbench/mvbench_episodic_reasoning.yaml | 19 ++ .../mvbench/mvbench_fine_grained_action.yaml | 8 + .../mvbench/mvbench_fine_grained_pose.yaml | 8 + .../mvbench/mvbench_moving_attribute.yaml | 8 + .../tasks/mvbench/mvbench_moving_count.yaml | 8 + .../mvbench/mvbench_moving_direction.yaml | 8 + .../mvbench/mvbench_object_existence.yaml | 8 + .../mvbench/mvbench_object_interaction.yaml | 8 + .../tasks/mvbench/mvbench_object_shuffle.yaml | 8 + .../mvbench/mvbench_scene_transition.yaml | 8 + .../tasks/mvbench/mvbench_state_change.yaml | 8 + .../mvbench/mvbench_unexpected_action.yaml | 8 + lmms_eval/tasks/mvbench/utils.py | 194 ++++++++++++++++++ lmms_eval/tasks/nextqa/nextqa_oe_test.yaml | 2 +- lmms_eval/tasks/nextqa/nextqa_oe_val.yaml | 2 +- lmms_eval/tasks/nextqa/utils.py | 24 +-- .../nocaps/_default_template_nocaps_yaml | 2 +- lmms_eval/tasks/nocaps/utils.py | 4 +- .../tasks/ok_vqa/_default_template_vqa_yaml | 2 +- lmms_eval/tasks/ok_vqa/utils.py | 14 +- .../test/_default_template_yaml | 2 +- lmms_eval/tasks/perceptiontest/test/utils.py | 14 +- .../perceptiontest/val/_default_template_yaml | 2 +- lmms_eval/tasks/perceptiontest/val/utils.py | 14 +- lmms_eval/tasks/qbench/abench_dev.yaml | 2 +- lmms_eval/tasks/qbench/qbench2_dev.yaml | 2 +- lmms_eval/tasks/qbench/qbench_dev.yaml | 2 +- lmms_eval/tasks/qbench/utils.py | 6 +- lmms_eval/tasks/realworldqa/realworldqa.yaml | 2 +- lmms_eval/tasks/realworldqa/utils.py | 14 +- lmms_eval/tasks/scienceqa/scienceqa.yaml | 2 +- lmms_eval/tasks/scienceqa/scienceqa_img.yaml | 2 +- lmms_eval/tasks/scienceqa/utils.py | 12 +- lmms_eval/tasks/seedbench_2/seedbench_2.yaml | 5 +- lmms_eval/tasks/stvqa/stvqa.yaml | 2 +- lmms_eval/tasks/stvqa/utils.py | 6 +- .../tasks/tempcompass/_default_template_yaml | 2 +- lmms_eval/tasks/tempcompass/utils.py | 56 ++--- .../textcaps/_default_template_textcaps_yaml | 2 +- lmms_eval/tasks/textcaps/utils.py | 4 +- .../textvqa/_default_template_textvqa_yaml | 2 +- lmms_eval/tasks/textvqa/utils.py | 14 +- lmms_eval/tasks/vatex/utils.py | 8 +- lmms_eval/tasks/vatex/vatex_test.yaml | 2 +- lmms_eval/tasks/vatex/vatex_val_zh.yaml | 2 +- lmms_eval/tasks/vcr_wiki/utils.py | 10 +- .../tasks/vcr_wiki/vcr_wiki_en_easy.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_en_easy_100.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_en_easy_500.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_en_hard.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_en_hard_100.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_en_hard_500.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_zh_easy.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_zh_easy_100.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_zh_easy_500.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_zh_hard.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_zh_hard_100.yaml | 2 +- .../tasks/vcr_wiki/vcr_wiki_zh_hard_500.yaml | 2 +- lmms_eval/tasks/vibe_eval/utils.py | 10 +- lmms_eval/tasks/vibe_eval/vibe_eval.yaml | 2 +- .../_default_template_yaml | 2 +- .../tasks/video_detail_description/utils.py | 14 +- .../tasks/videochatgpt/_default_template_yaml | 2 +- lmms_eval/tasks/videochatgpt/utils.py | 28 +-- lmms_eval/tasks/videomme/utils.py | 95 +++++---- lmms_eval/tasks/videomme/videomme.yaml | 2 +- .../tasks/videomme/videomme_w_subtitle.yaml | 2 +- .../tasks/vitatecs/_default_template_yaml | 2 +- lmms_eval/tasks/vitatecs/utils.py | 14 +- .../vizwiz_vqa/_default_template_vqa_yaml | 2 +- lmms_eval/tasks/vizwiz_vqa/utils.py | 14 +- .../tasks/vqav2/_default_template_vqav2_yaml | 2 +- lmms_eval/tasks/vqav2/utils.py | 14 +- lmms_eval/tasks/wild_vision_bench/utils.py | 10 +- .../wild_vision_bench0617.yaml | 2 +- lmms_eval/tasks/worldqa/utils.py | 14 +- .../tasks/worldqa/worldqa_generation.yaml | 2 +- lmms_eval/tasks/worldqa/worldqa_mc.yaml | 2 +- lmms_eval/tasks/worldqa/worldqa_mcppl.yaml | 2 +- lmms_eval/tasks/youcook2/utils.py | 6 +- lmms_eval/tasks/youcook2/youcook2_val.yaml | 2 +- 185 files changed, 925 insertions(+), 473 deletions(-) create mode 100644 lmms_eval/tasks/mvbench/_default_template.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_action_antonym.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_action_count.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_action_localization.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_action_prediction.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_action_sequence.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_character_order.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_counterfactual_inference.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_egocentric_navigation.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_fine_grained_action.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_fine_grained_pose.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_moving_attribute.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_moving_count.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_moving_direction.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_object_existence.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_object_interaction.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_object_shuffle.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_scene_transition.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_state_change.yaml create mode 100644 lmms_eval/tasks/mvbench/mvbench_unexpected_action.yaml create mode 100644 lmms_eval/tasks/mvbench/utils.py diff --git a/lmms_eval/api/instance.py b/lmms_eval/api/instance.py index 41875358d..56285ec0d 100755 --- a/lmms_eval/api/instance.py +++ b/lmms_eval/api/instance.py @@ -19,7 +19,7 @@ class Instance: def __post_init__(self) -> None: # unpack metadata field - self.task_name, self.doc_id, self.repeats = self.metadata + self.task_name, self.doc_id, self.repeats = self.metadata["task"], self.metadata["doc_id"], self.metadata["repeats"] @property def args(self): diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index f488bb224..af8e81b4e 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -4,9 +4,10 @@ import json import os -import random import re +import random import shutil +import inspect import subprocess from collections.abc import Callable from dataclasses import dataclass, field, asdict @@ -95,7 +96,7 @@ class TaskConfig(dict): metadata: Union[str, list] = None # by default, not used in the code. allows for users to pass arbitrary info to tasks - model_specific_prompt_kwargs: dict = None + lmms_eval_specific_kwargs: dict = None model_specific_generation_kwargs: dict = None model_specific_target_kwargs: dict = None @@ -376,7 +377,11 @@ def build_all_requests(self, limit=None, rank=None, world_size=None) -> None: fewshot_ctx = self.fewshot_context(doc_id, 0 if self.config.num_fewshot is None else self.config.num_fewshot, self.config.training_split if self.has_training_docs() else split) # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute - inst = self.construct_requests(doc_id=doc_id, ctx=fewshot_ctx, metadata=(self.config["task"], doc_id, self.config.repeats), split=split) + per_task_metadata = {"task": self.config["task"], "doc_id": doc_id, "repeats": self.config.repeats} + if self.config.metadata: + per_task_metadata.update(self.config.metadata) + + inst = self.construct_requests(doc_id=doc_id, ctx=fewshot_ctx, metadata=per_task_metadata, split=split) if not isinstance(inst, list): inst = [inst] @@ -630,12 +635,14 @@ def __init__(self, model_name) -> None: # TODO no super() call here eval_logger.warning(f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace') def _prepare_model_specific_config(self): - self.model_specific_prompt_kwargs = self.config.model_specific_prompt_kwargs - if self.model_specific_prompt_kwargs is not None: - if self.model_name in self.model_specific_prompt_kwargs: - self.model_specific_prompt_kwargs = self.model_specific_prompt_kwargs[self.model_name] - else: - self.model_specific_prompt_kwargs = self.model_specific_prompt_kwargs.get("default", None) + self.lmms_eval_specific_kwargs = self.config.lmms_eval_specific_kwargs + if self.lmms_eval_specific_kwargs is not None: + if self.model_name in self.lmms_eval_specific_kwargs: + self.lmms_eval_specific_kwargs = self.lmms_eval_specific_kwargs[self.model_name] + if "default" in self.lmms_eval_specific_kwargs: + self.lmms_eval_specific_kwargs.update(self.lmms_eval_specific_kwargs.get("default", {})) + if "dataset" in self.lmms_eval_specific_kwargs: + self.lmms_eval_specific_kwargs.update(self.lmms_eval_specific_kwargs.get("dataset", {})) self.model_specific_target_kwargs = self.config.model_specific_target_kwargs if self.model_specific_target_kwargs is not None: @@ -784,10 +791,17 @@ def _download_from_youtube(path): def unzip_video_data(zip_file): import zipfile + import os with zipfile.ZipFile(zip_file, "r") as zip_ref: - zip_ref.extractall(cache_dir) - eval_logger.info(f"Extracted all files from {zip_file} to {cache_dir}") + for file_info in zip_ref.infolist(): + target_path = os.path.join(cache_dir, file_info.filename) + if not os.path.exists(target_path): + zip_ref.extract(file_info, cache_dir) + else: + eval_logger.info(f"Skipping existing file: {target_path}") + + eval_logger.info(f"Extracted all files from {zip_file} to {cache_dir}") def untar_video_data(tar_file): import tarfile @@ -1006,8 +1020,8 @@ def doc_to_text(self, doc): return text_string elif callable(doc_to_text): return ( - doc_to_text(doc, self.model_specific_prompt_kwargs) - if self.model_specific_prompt_kwargs is not None + doc_to_text(doc, self.lmms_eval_specific_kwargs) + if self.lmms_eval_specific_kwargs is not None else doc_to_text( doc, ) @@ -1067,9 +1081,16 @@ def doc_to_visual(self, doc: dict) -> Union[int, str, list]: assert self.config.doc_to_visual in self.features # Single image. Still return a list for consistency. return [doc[self.config.doc_to_visual]] + elif callable(self.config.doc_to_visual): + return ( + self.config.doc_to_visual(doc, self.lmms_eval_specific_kwargs) + if self.lmms_eval_specific_kwargs is not None and len(inspect.signature(self.config.doc_to_visual).parameters) == 2 + else self.config.doc_to_visual( + doc, + ) + ) else: - assert callable(self.config.doc_to_visual) - return self.config.doc_to_visual(doc) + raise TypeError def doc_to_choice(self, doc: Any) -> List[str]: if self.config.doc_to_choice is None: diff --git a/lmms_eval/models/llava_onevision.py b/lmms_eval/models/llava_onevision.py index 4347bc519..4785c5db8 100644 --- a/lmms_eval/models/llava_onevision.py +++ b/lmms_eval/models/llava_onevision.py @@ -429,6 +429,7 @@ def _collate(x): # we group requests by their generation_kwargs, # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling # in the same batch. + metadata = requests[0].metadata re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True) chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1 @@ -452,6 +453,12 @@ def _collate(x): if len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__: # for multi image case, we treat per image aspect ratio as "pad" by default. self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad") eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") + + if "sample_frames" in metadata: + sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int) + visual = [visual[i] for i in sample_indices] + assert len(visual) == metadata["sample_frames"] + # if (len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__) and ("image_aspect_ratio" in gen_kwargs.keys()): # self._config.image_aspect_ratio = gen_kwargs["image_aspect_ratio"] # eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") diff --git a/lmms_eval/tasks/activitynetqa/_default_template_yaml b/lmms_eval/tasks/activitynetqa/_default_template_yaml index 69b410a84..7855f26bf 100644 --- a/lmms_eval/tasks/activitynetqa/_default_template_yaml +++ b/lmms_eval/tasks/activitynetqa/_default_template_yaml @@ -5,7 +5,7 @@ dataset_kwargs: force_download: False local_files_only: False cache_dir: activitynetqa -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: " Answer the question using a single word or phrase." diff --git a/lmms_eval/tasks/activitynetqa/utils.py b/lmms_eval/tasks/activitynetqa/utils.py index 0c77ff79e..a666ae1a1 100755 --- a/lmms_eval/tasks/activitynetqa/utils.py +++ b/lmms_eval/tasks/activitynetqa/utils.py @@ -61,15 +61,15 @@ def activitynetqa_doc_to_visual(doc): # This is the place where format the question -def activitynetqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def activitynetqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] raw_question = doc["question"] question = raw_question.capitalize() + "?" diff --git a/lmms_eval/tasks/ai2d/ai2d.yaml b/lmms_eval/tasks/ai2d/ai2d.yaml index 5221ff64b..7cf9cb013 100755 --- a/lmms_eval/tasks/ai2d/ai2d.yaml +++ b/lmms_eval/tasks/ai2d/ai2d.yaml @@ -8,7 +8,7 @@ doc_to_visual: !function utils.ai2d_doc_to_visual doc_to_text: !function utils.ai2d_doc_to_text doc_to_target: !function utils.ai2d_doc_to_target -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt_format: mcq pre_prompt: "" diff --git a/lmms_eval/tasks/ai2d/utils.py b/lmms_eval/tasks/ai2d/utils.py index a6e2368d6..e70f03d5b 100755 --- a/lmms_eval/tasks/ai2d/utils.py +++ b/lmms_eval/tasks/ai2d/utils.py @@ -3,24 +3,24 @@ import re -def ai2d_doc_to_text(doc, model_specific_prompt_kwargs=None): +def ai2d_doc_to_text(doc, lmms_eval_specific_kwargs=None): question, choices = doc["question"], doc["options"] len_choices = len(choices) - post_prompt = model_specific_prompt_kwargs["post_prompt"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if model_specific_prompt_kwargs["prompt_format"] == "mcq": + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if lmms_eval_specific_kwargs["prompt_format"] == "mcq": options = [chr(ord("A") + i) for i in range(len_choices)] choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) return f"{pre_prompt}{question}\n{choices_str}{post_prompt}" - elif model_specific_prompt_kwargs["prompt_format"] == "qa": + elif lmms_eval_specific_kwargs["prompt_format"] == "qa": options = "\n".join(choices) return f"{pre_prompt}{question}{options}{post_prompt}" - elif model_specific_prompt_kwargs["prompt_format"] == "mcq_xcomposer": + elif lmms_eval_specific_kwargs["prompt_format"] == "mcq_xcomposer": options = [chr(ord("A") + i) for i in range(len_choices)] choices_str = " ".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) return f"{pre_prompt}{question}\nContext: N/A\n{choices_str}{post_prompt}" else: - raise ValueError(f"Unknown prompt format: {model_specific_prompt_kwargs['prompt_format']}") + raise ValueError(f"Unknown prompt format: {lmms_eval_specific_kwargs['prompt_format']}") def ai2d_doc_to_visual(doc): diff --git a/lmms_eval/tasks/chartqa/chartqa.yaml b/lmms_eval/tasks/chartqa/chartqa.yaml index e8b7e82f2..2a42f4dfd 100755 --- a/lmms_eval/tasks/chartqa/chartqa.yaml +++ b/lmms_eval/tasks/chartqa/chartqa.yaml @@ -24,7 +24,7 @@ metric_list: higher_is_better: true metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question with a single word." diff --git a/lmms_eval/tasks/chartqa/utils.py b/lmms_eval/tasks/chartqa/utils.py index 99de0989c..cdba4c411 100755 --- a/lmms_eval/tasks/chartqa/utils.py +++ b/lmms_eval/tasks/chartqa/utils.py @@ -2,10 +2,10 @@ def chartqa_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def chartqa_doc_to_text(doc, model_specific_prompt_kwargs): +def chartqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/cvrr/_default_template_yaml b/lmms_eval/tasks/cvrr/_default_template_yaml index 333946a31..61465e966 100644 --- a/lmms_eval/tasks/cvrr/_default_template_yaml +++ b/lmms_eval/tasks/cvrr/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: cvrr-es -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/cvrr/utils.py b/lmms_eval/tasks/cvrr/utils.py index b2eb25c44..8ee5700bf 100755 --- a/lmms_eval/tasks/cvrr/utils.py +++ b/lmms_eval/tasks/cvrr/utils.py @@ -85,15 +85,15 @@ def cvrr_doc_to_visual(doc): # format the question -def cvrr_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def cvrr_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["Q"] diff --git a/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml b/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml index f673aeb7b..1a00f63dd 100644 --- a/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml +++ b/lmms_eval/tasks/detailcaps/_default_template_detailcaps_yaml @@ -1,3 +1,3 @@ -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt: "Describe this image in detail." \ No newline at end of file diff --git a/lmms_eval/tasks/detailcaps/utils.py b/lmms_eval/tasks/detailcaps/utils.py index 50c0aba25..1c6955e94 100644 --- a/lmms_eval/tasks/detailcaps/utils.py +++ b/lmms_eval/tasks/detailcaps/utils.py @@ -23,9 +23,9 @@ def detailcaps_doc_to_visual(doc): return [Image.open(io.BytesIO(doc["binary"])).convert("RGB")] -def detailcaps_doc_to_text(doc, model_specific_prompt_kwargs=None): +def detailcaps_doc_to_text(doc, lmms_eval_specific_kwargs=None): # question = "Please carefully observe the image and come up with a caption for the image" - return model_specific_prompt_kwargs["prompt"] + return lmms_eval_specific_kwargs["prompt"] def detailcaps_doc_to_target(doc): diff --git a/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml b/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml index 6e0cab68c..80acd7ba6 100755 --- a/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml +++ b/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml @@ -10,7 +10,7 @@ generation_kwargs: max_new_tokens: 32 temperature: 0 do_sample: False -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/docvqa/utils.py b/lmms_eval/tasks/docvqa/utils.py index bb17aa03d..977fbd109 100755 --- a/lmms_eval/tasks/docvqa/utils.py +++ b/lmms_eval/tasks/docvqa/utils.py @@ -11,10 +11,10 @@ def docvqa_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def docvqa_doc_to_text(doc, model_specific_prompt_kwargs): +def docvqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/egoschema/_default_template_yaml b/lmms_eval/tasks/egoschema/_default_template_yaml index 8e0306454..29b214702 100644 --- a/lmms_eval/tasks/egoschema/_default_template_yaml +++ b/lmms_eval/tasks/egoschema/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: egoschema -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/egoschema/utils.py b/lmms_eval/tasks/egoschema/utils.py index d2c1f66c8..fc5ac8d5b 100755 --- a/lmms_eval/tasks/egoschema/utils.py +++ b/lmms_eval/tasks/egoschema/utils.py @@ -45,15 +45,15 @@ def egoschema_doc_to_visual(doc): # This is the place where you format your question -def egoschema_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def egoschema_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["question"] if "option" in doc: diff --git a/lmms_eval/tasks/ferret/ferret.yaml b/lmms_eval/tasks/ferret/ferret.yaml index 249b711bf..680969114 100755 --- a/lmms_eval/tasks/ferret/ferret.yaml +++ b/lmms_eval/tasks/ferret/ferret.yaml @@ -33,7 +33,7 @@ metric_list: metadata: version: 0.0 gpt_eval_model_name: "gpt-4-0314" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/ferret/utils.py b/lmms_eval/tasks/ferret/utils.py index bb96b4a19..4779ab8b1 100755 --- a/lmms_eval/tasks/ferret/utils.py +++ b/lmms_eval/tasks/ferret/utils.py @@ -106,11 +106,11 @@ def ferret_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def ferret_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def ferret_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") question = f"{pre_prompt}{doc['question']}{post_prompt}" return question diff --git a/lmms_eval/tasks/gqa/gqa.yaml b/lmms_eval/tasks/gqa/gqa.yaml index 5d06c37bc..64f7b22f5 100755 --- a/lmms_eval/tasks/gqa/gqa.yaml +++ b/lmms_eval/tasks/gqa/gqa.yaml @@ -23,7 +23,7 @@ metric_list: metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/gqa/utils.py b/lmms_eval/tasks/gqa/utils.py index 5d74d6fb7..c6d91f6e4 100755 --- a/lmms_eval/tasks/gqa/utils.py +++ b/lmms_eval/tasks/gqa/utils.py @@ -16,8 +16,8 @@ def gqa_doc_to_visual(doc): return [image] -def gqa_doc_to_text(doc, model_specific_prompt_kwargs): +def gqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/gqa_ru/gqa_ru.yaml b/lmms_eval/tasks/gqa_ru/gqa_ru.yaml index 2a3d10972..452f40c1e 100644 --- a/lmms_eval/tasks/gqa_ru/gqa_ru.yaml +++ b/lmms_eval/tasks/gqa_ru/gqa_ru.yaml @@ -23,7 +23,7 @@ metric_list: metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nОтветь одним словом." \ No newline at end of file diff --git a/lmms_eval/tasks/gqa_ru/utils.py b/lmms_eval/tasks/gqa_ru/utils.py index 9c1acb9ce..e0e631d6c 100644 --- a/lmms_eval/tasks/gqa_ru/utils.py +++ b/lmms_eval/tasks/gqa_ru/utils.py @@ -16,8 +16,8 @@ def gqa_doc_to_visual(doc): return [image] -def gqa_doc_to_text(doc, model_specific_prompt_kwargs): +def gqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/hallusion_bench/evaluate_hb.py b/lmms_eval/tasks/hallusion_bench/evaluate_hb.py index f35c67a24..9c03a943b 100755 --- a/lmms_eval/tasks/hallusion_bench/evaluate_hb.py +++ b/lmms_eval/tasks/hallusion_bench/evaluate_hb.py @@ -14,11 +14,11 @@ from loguru import logger as eval_logger -def hb_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def hb_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{doc['question']}{post_prompt}" diff --git a/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml b/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml index d5a485bac..ed6985d9d 100755 --- a/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml +++ b/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml @@ -8,7 +8,7 @@ doc_to_visual: !function evaluate_hb.hb_doc_to_visual doc_to_text: !function evaluate_hb.hb_doc_to_text doc_to_target: "gt_answer_details" process_results: !function evaluate_hb.hb_process_results -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml b/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml index 66487888c..c81dab2b6 100755 --- a/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml +++ b/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml @@ -10,7 +10,7 @@ generation_kwargs: max_new_tokens: 32 temperature: 0 do_sample: False -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" statement: "Given a set of images and a question, please provide the answer to the question.\n" diff --git a/lmms_eval/tasks/iconqa/utils.py b/lmms_eval/tasks/iconqa/utils.py index de99e8047..a85597b48 100755 --- a/lmms_eval/tasks/iconqa/utils.py +++ b/lmms_eval/tasks/iconqa/utils.py @@ -24,7 +24,7 @@ def doc_to_visual(doc): return image_list -def doc_to_text(doc, model_specific_prompt_kwargs): +def doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] ques_type = doc["ques_type"] options_prompt = [] @@ -34,7 +34,7 @@ def doc_to_text(doc, model_specific_prompt_kwargs): options_prompt.append("The second image.") options_str = options_to_str(options_prompt) - full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['options_statement'].format(question=question, options=options_str)}" + full_prompt = f"{lmms_eval_specific_kwargs['pre_prompt']}{lmms_eval_specific_kwargs['statement']}{lmms_eval_specific_kwargs['options_statement'].format(question=question, options=options_str)}" elif ques_type == "choose_txt": choices = doc["choices"].split(",") @@ -42,10 +42,10 @@ def doc_to_text(doc, model_specific_prompt_kwargs): options_prompt.append(f"{choice}") options_str = options_to_str(options_prompt) - full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['options_statement'].format(question=question, options=options_str)}" + full_prompt = f"{lmms_eval_specific_kwargs['pre_prompt']}{lmms_eval_specific_kwargs['statement']}{lmms_eval_specific_kwargs['options_statement'].format(question=question, options=options_str)}" elif ques_type == "fill_in_blank": - full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['freeform_statement'].format(question=question)}" + full_prompt = f"{lmms_eval_specific_kwargs['pre_prompt']}{lmms_eval_specific_kwargs['statement']}{lmms_eval_specific_kwargs['freeform_statement'].format(question=question)}" return full_prompt diff --git a/lmms_eval/tasks/ii_bench/ii_bench.yaml b/lmms_eval/tasks/ii_bench/ii_bench.yaml index 482eb9cf7..83e859baf 100755 --- a/lmms_eval/tasks/ii_bench/ii_bench.yaml +++ b/lmms_eval/tasks/ii_bench/ii_bench.yaml @@ -13,7 +13,7 @@ process_results: !function utils.ii_bench_process_results metric_list: - metric: submission aggregation: !function utils.ii_bench_aggregate_submissions -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "Instruction: Please try to answer the single-answer multiple choice question below based on the picture provided.\n" post_prompt: "\nAnswer:" diff --git a/lmms_eval/tasks/ii_bench/utils.py b/lmms_eval/tasks/ii_bench/utils.py index 8b30c12a9..756617117 100755 --- a/lmms_eval/tasks/ii_bench/utils.py +++ b/lmms_eval/tasks/ii_bench/utils.py @@ -14,10 +14,10 @@ (F) {}""" -def ii_bench_doc_to_text(doc, model_specific_prompt_kwargs): +def ii_bench_doc_to_text(doc, lmms_eval_specific_kwargs): question = PROMPT.format(doc["question"], doc["option1"], doc["option2"], doc["option3"], doc["option4"], doc["option5"], doc["option6"]) - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml b/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml index 22d69ed9f..c8153273b 100755 --- a/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml +++ b/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml @@ -9,7 +9,7 @@ generation_kwargs: max_new_tokens: 32 temperature: 0 do_sample: False -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." \ No newline at end of file diff --git a/lmms_eval/tasks/infovqa/utils.py b/lmms_eval/tasks/infovqa/utils.py index 9907a6cb2..9aaea3715 100755 --- a/lmms_eval/tasks/infovqa/utils.py +++ b/lmms_eval/tasks/infovqa/utils.py @@ -11,10 +11,10 @@ def infovqa_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def infovqa_doc_to_text(doc, model_specific_prompt_kwargs): +def infovqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml b/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml index 5281ad1de..b2459edf4 100755 --- a/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml +++ b/lmms_eval/tasks/internal_eval/_default_template_internal_eval_yaml @@ -1,4 +1,4 @@ -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/internal_eval/utils.py b/lmms_eval/tasks/internal_eval/utils.py index aab8ab3c4..cc6efc932 100755 --- a/lmms_eval/tasks/internal_eval/utils.py +++ b/lmms_eval/tasks/internal_eval/utils.py @@ -1,7 +1,7 @@ -def doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: +def doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: return doc["question"] question = doc["question"] - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/live_bench/live_bench.yaml b/lmms_eval/tasks/live_bench/live_bench.yaml index 901a12a87..1253105ee 100755 --- a/lmms_eval/tasks/live_bench/live_bench.yaml +++ b/lmms_eval/tasks/live_bench/live_bench.yaml @@ -2,6 +2,7 @@ group: live_bench task: - live_bench_2406 - live_bench_2407 + metadata: api_type : openai eval_with_mini: false diff --git a/lmms_eval/tasks/live_bench/live_bench_template_yaml b/lmms_eval/tasks/live_bench/live_bench_template_yaml index e3d9877ed..2cea13279 100644 --- a/lmms_eval/tasks/live_bench/live_bench_template_yaml +++ b/lmms_eval/tasks/live_bench/live_bench_template_yaml @@ -22,7 +22,8 @@ metric_list: # - metric: gpt4_eval_score_mini # aggregation: !function utils.livebench_aggregate_results # higher_is_better: true -model_specific_prompt_kwargs: + +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/live_bench/utils.py b/lmms_eval/tasks/live_bench/utils.py index 858700295..524aa9acb 100644 --- a/lmms_eval/tasks/live_bench/utils.py +++ b/lmms_eval/tasks/live_bench/utils.py @@ -148,11 +148,11 @@ def livebench_doc_to_visual(doc): return img_list -def livebench_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def livebench_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{doc['question']}{post_prompt}" diff --git a/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml b/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml index b104bcb79..352ef50fd 100755 --- a/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml +++ b/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml @@ -32,7 +32,7 @@ metric_list: metadata: version: 0.0 gpt_eval_model_name: "gpt-4-0314" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/llava-bench-coco/utils.py b/lmms_eval/tasks/llava-bench-coco/utils.py index 08bf25155..b7ff8a800 100755 --- a/lmms_eval/tasks/llava-bench-coco/utils.py +++ b/lmms_eval/tasks/llava-bench-coco/utils.py @@ -109,11 +109,11 @@ def llava_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def llava_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{doc['question']}{post_prompt}" diff --git a/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml b/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml index c0db28e18..785ccb830 100755 --- a/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml +++ b/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml @@ -33,7 +33,7 @@ metric_list: metadata: version: 0.0 gpt_eval_model_name: "gpt-4-0613" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/llava-in-the-wild/utils.py b/lmms_eval/tasks/llava-in-the-wild/utils.py index 25bf37a33..1a8daf899 100755 --- a/lmms_eval/tasks/llava-in-the-wild/utils.py +++ b/lmms_eval/tasks/llava-in-the-wild/utils.py @@ -106,11 +106,11 @@ def llava_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def llava_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{doc['question']}{post_prompt}" diff --git a/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml b/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml index 9a6bf8e43..602f79a49 100644 --- a/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml +++ b/lmms_eval/tasks/llava_interleave_bench/in_domain.yaml @@ -20,7 +20,7 @@ generation_kwargs: do_sample: False image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy. -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: oe_post_prompt: "" mcq_post_prompt: "" diff --git a/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml b/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml index 5b3dec18b..38627c236 100644 --- a/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml +++ b/lmms_eval/tasks/llava_interleave_bench/multi_view_in_domain.yaml @@ -20,7 +20,7 @@ generation_kwargs: do_sample: False image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy. -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: oe_post_prompt: "" mcq_post_prompt: "" diff --git a/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml b/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml index 47c3a9518..76f217213 100644 --- a/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml +++ b/lmms_eval/tasks/llava_interleave_bench/out_of_domain.yaml @@ -20,7 +20,7 @@ generation_kwargs: do_sample: False image_aspect_ratio: "pad" # for multi-image, we treat each image as original aspect ratio without anyres strategy. -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: oe_post_prompt: "" mcq_post_prompt: "" diff --git a/lmms_eval/tasks/llava_interleave_bench/utils.py b/lmms_eval/tasks/llava_interleave_bench/utils.py index 3c871d3ce..955494274 100644 --- a/lmms_eval/tasks/llava_interleave_bench/utils.py +++ b/lmms_eval/tasks/llava_interleave_bench/utils.py @@ -56,17 +56,17 @@ def doc_to_visual(doc): # This is the place where you format your question -def doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} oe_post_prompt = "" - if "oe_post_prompt" in model_specific_prompt_kwargs: - oe_post_prompt = model_specific_prompt_kwargs["oe_post_prompt"] + if "oe_post_prompt" in lmms_eval_specific_kwargs: + oe_post_prompt = lmms_eval_specific_kwargs["oe_post_prompt"] mcq_post_prompt = "" - if "mcq_post_prompt" in model_specific_prompt_kwargs: - mcq_post_prompt = model_specific_prompt_kwargs["mcq_post_prompt"] + if "mcq_post_prompt" in lmms_eval_specific_kwargs: + mcq_post_prompt = lmms_eval_specific_kwargs["mcq_post_prompt"] user_prompt = doc["question"] @@ -80,9 +80,9 @@ def doc_to_text(doc, model_specific_prompt_kwargs=None): return user_prompt -def doc_to_text_conversation(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def doc_to_text_conversation(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} conversations = doc["conversations"] @@ -94,9 +94,9 @@ def doc_to_text_conversation(doc, model_specific_prompt_kwargs=None): return user_prompt -def doc_to_text_multi_turn(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def doc_to_text_multi_turn(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} return doc["conversations"] diff --git a/lmms_eval/tasks/llava_wilder/llava_wilder_small.yaml b/lmms_eval/tasks/llava_wilder/llava_wilder_small.yaml index 1d112ee2b..d0d4049dc 100644 --- a/lmms_eval/tasks/llava_wilder/llava_wilder_small.yaml +++ b/lmms_eval/tasks/llava_wilder/llava_wilder_small.yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True task: "llava_wilder_small" test_split: test -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/llava_wilder/utils.py b/lmms_eval/tasks/llava_wilder/utils.py index 04bb82738..2dfe799e5 100644 --- a/lmms_eval/tasks/llava_wilder/utils.py +++ b/lmms_eval/tasks/llava_wilder/utils.py @@ -143,11 +143,11 @@ def llava_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def llava_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{doc['Question']}{post_prompt}" diff --git a/lmms_eval/tasks/longvideobench/longvideobench_val_i.yaml b/lmms_eval/tasks/longvideobench/longvideobench_val_i.yaml index decb12d6a..417d8c8b1 100644 --- a/lmms_eval/tasks/longvideobench/longvideobench_val_i.yaml +++ b/lmms_eval/tasks/longvideobench/longvideobench_val_i.yaml @@ -21,7 +21,7 @@ metric_list: aggregation: !function utils.longvideobench_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "Answer with the option's letter from the given choices directly.\n" diff --git a/lmms_eval/tasks/longvideobench/longvideobench_val_v.yaml b/lmms_eval/tasks/longvideobench/longvideobench_val_v.yaml index e926ed459..cf1c00b92 100644 --- a/lmms_eval/tasks/longvideobench/longvideobench_val_v.yaml +++ b/lmms_eval/tasks/longvideobench/longvideobench_val_v.yaml @@ -21,7 +21,7 @@ metric_list: aggregation: !function utils.longvideobench_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "Answer with the option's letter from the given choices directly.\n" diff --git a/lmms_eval/tasks/longvideobench/utils.py b/lmms_eval/tasks/longvideobench/utils.py index b71e30864..5d1bad20c 100644 --- a/lmms_eval/tasks/longvideobench/utils.py +++ b/lmms_eval/tasks/longvideobench/utils.py @@ -114,7 +114,7 @@ def insert_subtitles_into_frames(frame_timestamps, subtitles, starting_timestamp return "\n".join(interleaved_list) -def longvideobench_doc_to_text(doc, model_specific_prompt_kwargs): +def longvideobench_doc_to_text(doc, lmms_eval_specific_kwargs): candidates = [] for i in range(5): @@ -123,10 +123,10 @@ def longvideobench_doc_to_text(doc, model_specific_prompt_kwargs): candidates.append(candidate) question = doc["question"] + "\n" + "\n".join([". ".join([chr(ord("A") + i), candidate]) for i, candidate in enumerate(candidates)]) - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] - if model_specific_prompt_kwargs.get("insert_interleave_subtitles", False): + if lmms_eval_specific_kwargs.get("insert_interleave_subtitles", False): with open(Path(__file__).parent / "longvideobench_val_i.yaml", "r") as f: raw_data = f.readlines() safe_data = [] diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini.yaml index 62cd8e157..b6f060940 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml index 7ff121e90..62159295f 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml index 2137e354a..1ea1feaf6 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml index 42ceb5ee8..2916b5735 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml index fe66a40a7..72b0653b6 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml index cb922997e..0e6eaa246 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml index 72df73690..ab7934966 100644 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml +++ b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml @@ -25,7 +25,7 @@ metric_list: aggregation: !function utils.mathverse_aggregate_results_submission higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can also be "custom-prompt" query_type: "query_wo" # now only support query_wo diff --git a/lmms_eval/tasks/mathverse/utils.py b/lmms_eval/tasks/mathverse/utils.py index dde6eeb27..1f1302e52 100644 --- a/lmms_eval/tasks/mathverse/utils.py +++ b/lmms_eval/tasks/mathverse/utils.py @@ -27,7 +27,7 @@ def mathverse_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mathverse_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mathverse_doc_to_text(doc, lmms_eval_specific_kwargs=None): problem = { "question": doc["question"], "answer": doc["answer"] if "answer" in doc else None, @@ -37,7 +37,7 @@ def mathverse_doc_to_text(doc, model_specific_prompt_kwargs=None): "problem_version": doc["problem_version"], } query_prompt = mathverse_evaluator.create_one_query( - problem, examples=None, shot_num=0, shot_type=model_specific_prompt_kwargs["shot_type"], hint=model_specific_prompt_kwargs.get("hint", None), query_type=model_specific_prompt_kwargs["query_type"] + problem, examples=None, shot_num=0, shot_type=lmms_eval_specific_kwargs["shot_type"], hint=lmms_eval_specific_kwargs.get("hint", None), query_type=lmms_eval_specific_kwargs["query_type"] ) return query_prompt @@ -79,7 +79,7 @@ def mathverse_aggregate_results_submission(results, args, *, calculate_gain=Fals def mathverse_aggregate_results_eval(results, args, *, calculate_gain=False, random_scores=None): split_flag = results[0]["metadata"]["split"] - problem_version = results[0]["problem_version"].lower().replace(" ", "_") + problem_version = results[0]["metadata"]["problem_version"].lower().replace(" ", "_") # save the result first, in case the gpt evaluation fails path = generate_submission_file(f"mathverse_{split_flag}_{problem_version}_results.json", args) with open(path, "w") as f: diff --git a/lmms_eval/tasks/mathvista/mathvista_test.yaml b/lmms_eval/tasks/mathvista/mathvista_test.yaml index 35259bd26..ce732db60 100755 --- a/lmms_eval/tasks/mathvista/mathvista_test.yaml +++ b/lmms_eval/tasks/mathvista/mathvista_test.yaml @@ -19,6 +19,6 @@ metric_list: aggregation: !function utils.mathvista_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "reason-first" # can be "reason-first", "solution", "step-by-step" \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml index a801dc6db..aab4f53c9 100755 --- a/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml +++ b/lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml @@ -19,7 +19,7 @@ metric_list: aggregation: !function utils.mathvista_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "step-by-step" # can be "reason-first", "solution", "step-by-step" shot: 0 diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml index 8dc819716..bcbf2d9ad 100755 --- a/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml +++ b/lmms_eval/tasks/mathvista/mathvista_testmini_format.yaml @@ -19,7 +19,7 @@ metric_list: aggregation: !function utils.mathvista_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step" shot: 0 diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml index 8eb77b88d..d5e8bec28 100755 --- a/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml +++ b/lmms_eval/tasks/mathvista/mathvista_testmini_solution.yaml @@ -19,7 +19,7 @@ metric_list: aggregation: !function utils.mathvista_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: shot_type: "solution" # can be "reason-first", "solution", "step-by-step" shot: 0 diff --git a/lmms_eval/tasks/mathvista/utils.py b/lmms_eval/tasks/mathvista/utils.py index 4f8ad21ba..d33f0c1c9 100755 --- a/lmms_eval/tasks/mathvista/utils.py +++ b/lmms_eval/tasks/mathvista/utils.py @@ -25,7 +25,7 @@ def mathvista_doc_to_visual(doc): return [doc["decoded_image"].convert("RGB")] -def mathvista_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mathvista_doc_to_text(doc, lmms_eval_specific_kwargs=None): problem = { "question_type": doc["question_type"], "answer_type": doc["answer_type"], @@ -39,10 +39,10 @@ def mathvista_doc_to_text(doc, model_specific_prompt_kwargs=None): } query_prompt = mathvista_evaluator.create_one_query( problem, - shot_num=model_specific_prompt_kwargs["shot"], - shot_type=model_specific_prompt_kwargs["shot_type"], - use_caption=model_specific_prompt_kwargs["use_caption"], - use_ocr=model_specific_prompt_kwargs["use_ocr"], + shot_num=lmms_eval_specific_kwargs["shot"], + shot_type=lmms_eval_specific_kwargs["shot_type"], + use_caption=lmms_eval_specific_kwargs["use_caption"], + use_ocr=lmms_eval_specific_kwargs["use_ocr"], ) return query_prompt diff --git a/lmms_eval/tasks/mlvu/utils.py b/lmms_eval/tasks/mlvu/utils.py index 9829ee8af..cb27aa3f9 100644 --- a/lmms_eval/tasks/mlvu/utils.py +++ b/lmms_eval/tasks/mlvu/utils.py @@ -39,7 +39,7 @@ def mlvu_doc_to_visual(doc): return [video_path] -def mlvu_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mlvu_doc_to_text(doc, lmms_eval_specific_kwargs=None): # option_prompt="Carefully watch this video and pay attention to every detail. Based on your observations, select the best option that accurately addresses the question." option_prompt = "" question = doc["question"] + "\nOnly give the best option.\n" diff --git a/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml b/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml index 6a699725a..7f2ecc422 100755 --- a/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml +++ b/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml @@ -13,7 +13,7 @@ generation_kwargs: num_beams: 1 do_sample: false process_results: !function cn_utils.mmbench_process_results -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\n请直接使用所提供的选项字母作为答案回答。" diff --git a/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml b/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml index 060bd5d1a..71a6df9ea 100755 --- a/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml +++ b/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml @@ -2,7 +2,7 @@ dataset_path: lmms-lab/MMBench dataset_kwargs: token: True doc_to_target: "answer" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly." diff --git a/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml b/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml index 993cd52ad..2caaeb02b 100644 --- a/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml +++ b/lmms_eval/tasks/mmbench/_default_template_mmbench_ru_yaml @@ -2,7 +2,7 @@ dataset_path: deepvk/MMBench-ru dataset_kwargs: token: True doc_to_target: "answer" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nВыбери правильный вариант ответа буквой." diff --git a/lmms_eval/tasks/mmbench/cc_utils.py b/lmms_eval/tasks/mmbench/cc_utils.py index 8cd6665de..98748efae 100755 --- a/lmms_eval/tasks/mmbench/cc_utils.py +++ b/lmms_eval/tasks/mmbench/cc_utils.py @@ -38,7 +38,7 @@ def mmbench_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mmbench_cn_cc_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mmbench_cn_cc_doc_to_text(doc, lmms_eval_specific_kwargs=None): option_candidate = ["A", "B", "C", "D", "E"] options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) @@ -55,8 +55,8 @@ def mmbench_cn_cc_doc_to_text(doc, model_specific_prompt_kwargs=None): query_prompt = f"{data['question']} {data['options']}" - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" + if lmms_eval_specific_kwargs: + query_prompt = f"{query_prompt}\n{lmms_eval_specific_kwargs['post_prompt']}" return query_prompt diff --git a/lmms_eval/tasks/mmbench/cn_utils.py b/lmms_eval/tasks/mmbench/cn_utils.py index 2a767d7ad..e36aa7862 100755 --- a/lmms_eval/tasks/mmbench/cn_utils.py +++ b/lmms_eval/tasks/mmbench/cn_utils.py @@ -40,7 +40,7 @@ def mmbench_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mmbench_doc_to_text(doc, lmms_eval_specific_kwargs=None): option_candidate = ["A", "B", "C", "D", "E"] options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) @@ -60,8 +60,8 @@ def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): query_prompt = f"{data['hint']} {data['question']} {data['options']}" if pd.notna(data["hint"]) else f"{data['question']} {data['options']}" - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" + if lmms_eval_specific_kwargs: + query_prompt = f"{query_prompt}\n{lmms_eval_specific_kwargs['post_prompt']}" return query_prompt diff --git a/lmms_eval/tasks/mmbench/en_utils.py b/lmms_eval/tasks/mmbench/en_utils.py index f0b5e9f88..4b138f13f 100755 --- a/lmms_eval/tasks/mmbench/en_utils.py +++ b/lmms_eval/tasks/mmbench/en_utils.py @@ -39,7 +39,7 @@ def mmbench_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mmbench_doc_to_text(doc, lmms_eval_specific_kwargs=None): option_candidate = ["A", "B", "C", "D", "E"] options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) @@ -59,8 +59,8 @@ def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): query_prompt = f"{data['hint']} {data['question']} {data['options']}" if pd.notna(data["hint"]) and data["hint"] != "nan" else f"{data['question']} {data['options']}" - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" + if lmms_eval_specific_kwargs: + query_prompt = f"{query_prompt}\n{lmms_eval_specific_kwargs['post_prompt']}" return query_prompt diff --git a/lmms_eval/tasks/mmbench/mmbench_cc.yaml b/lmms_eval/tasks/mmbench/mmbench_cc.yaml index 9b7b3cb16..2ce01ead6 100755 --- a/lmms_eval/tasks/mmbench/mmbench_cc.yaml +++ b/lmms_eval/tasks/mmbench/mmbench_cc.yaml @@ -25,7 +25,7 @@ metadata: version: 0.0 gpt_eval_model_name: "gpt-3.5-turbo-0613" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\n请直接使用所提供的选项字母作为答案回答。" diff --git a/lmms_eval/tasks/mmbench/ru_utils.py b/lmms_eval/tasks/mmbench/ru_utils.py index 3ff515c7c..6904b06fe 100644 --- a/lmms_eval/tasks/mmbench/ru_utils.py +++ b/lmms_eval/tasks/mmbench/ru_utils.py @@ -39,7 +39,7 @@ def mmbench_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mmbench_doc_to_text(doc, lmms_eval_specific_kwargs=None): option_candidate = ["A", "B", "C", "D", "E"] options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) @@ -59,8 +59,8 @@ def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): query_prompt = f"{data['hint']} {data['question']} {data['options']}" if pd.notna(data["hint"]) and data["hint"] != "nan" else f"{data['question']} {data['options']}" - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" + if lmms_eval_specific_kwargs: + query_prompt = f"{query_prompt}\n{lmms_eval_specific_kwargs['post_prompt']}" return query_prompt diff --git a/lmms_eval/tasks/mme/mme.yaml b/lmms_eval/tasks/mme/mme.yaml index 489745aac..e2ffd29d9 100755 --- a/lmms_eval/tasks/mme/mme.yaml +++ b/lmms_eval/tasks/mme/mme.yaml @@ -23,7 +23,7 @@ metric_list: - metric: mme_cognition_score aggregation: !function utils.mme_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/mme/utils.py b/lmms_eval/tasks/mme/utils.py index 5aa4ae18c..ce4f9f1f9 100755 --- a/lmms_eval/tasks/mme/utils.py +++ b/lmms_eval/tasks/mme/utils.py @@ -38,14 +38,14 @@ def mme_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mme_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mme_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["question"].strip() - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": + if "pre_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["pre_prompt"] != "": question = question.replace(replace_prompt, "") - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": + question = f"{lmms_eval_specific_kwargs['pre_prompt']}{question}" + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"] != "": question = question.replace(replace_prompt, "") - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + question = f"{question}{lmms_eval_specific_kwargs['post_prompt']}" return question diff --git a/lmms_eval/tasks/mmstar/mmstar.yaml b/lmms_eval/tasks/mmstar/mmstar.yaml index 338efe2f4..128ae5113 100644 --- a/lmms_eval/tasks/mmstar/mmstar.yaml +++ b/lmms_eval/tasks/mmstar/mmstar.yaml @@ -32,7 +32,7 @@ metric_list: - metric: average aggregation: !function utils.mmstar_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly" diff --git a/lmms_eval/tasks/mmstar/utils.py b/lmms_eval/tasks/mmstar/utils.py index b1b2c6f02..5d3a8f6ec 100644 --- a/lmms_eval/tasks/mmstar/utils.py +++ b/lmms_eval/tasks/mmstar/utils.py @@ -26,14 +26,14 @@ def mmstar_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def mmstar_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mmstar_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["question"].strip() - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": + if "pre_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["pre_prompt"] != "": question = question.replace(replace_prompt, "") - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": + question = f"{lmms_eval_specific_kwargs['pre_prompt']}{question}" + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"] != "": question = question.replace(replace_prompt, "") - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + question = f"{question}{lmms_eval_specific_kwargs['post_prompt']}" return question diff --git a/lmms_eval/tasks/mmupd/mmaad_base.yaml b/lmms_eval/tasks/mmupd/mmaad_base.yaml index 9a66b3e8e..bfa50a819 100644 --- a/lmms_eval/tasks/mmupd/mmaad_base.yaml +++ b/lmms_eval/tasks/mmupd/mmaad_base.yaml @@ -1,7 +1,7 @@ task: "mmaad_base" test_split: test dataset_name: mmaad_base -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\n" diff --git a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml b/lmms_eval/tasks/mmupd/mmaad_instruction.yaml index 30a3bac15..20c83804b 100644 --- a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml +++ b/lmms_eval/tasks/mmupd/mmaad_instruction.yaml @@ -1,7 +1,7 @@ task: "mmaad_instruction" test_split: test dataset_name: mmaad_base -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." diff --git a/lmms_eval/tasks/mmupd/mmaad_option.yaml b/lmms_eval/tasks/mmupd/mmaad_option.yaml index f110b8228..8a7c28306 100644 --- a/lmms_eval/tasks/mmupd/mmaad_option.yaml +++ b/lmms_eval/tasks/mmupd/mmaad_option.yaml @@ -1,7 +1,7 @@ task: "mmaad_option" test_split: test dataset_name: mmaad_option -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly." diff --git a/lmms_eval/tasks/mmupd/mmiasd_base.yaml b/lmms_eval/tasks/mmupd/mmiasd_base.yaml index 6a9159fdb..36e1b18ae 100644 --- a/lmms_eval/tasks/mmupd/mmiasd_base.yaml +++ b/lmms_eval/tasks/mmupd/mmiasd_base.yaml @@ -1,7 +1,7 @@ task: "mmiasd_base" test_split: test dataset_name: mmiasd_base -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\n" diff --git a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml b/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml index d0ac3a005..503de6d29 100644 --- a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml +++ b/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml @@ -1,7 +1,7 @@ task: "mmiasd_instruction" test_split: test dataset_name: mmiasd_base -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." diff --git a/lmms_eval/tasks/mmupd/mmiasd_option.yaml b/lmms_eval/tasks/mmupd/mmiasd_option.yaml index a1a1aeadd..16274c50c 100644 --- a/lmms_eval/tasks/mmupd/mmiasd_option.yaml +++ b/lmms_eval/tasks/mmupd/mmiasd_option.yaml @@ -1,7 +1,7 @@ task: "mmiasd_option" test_split: test dataset_name: mmiasd_option -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly." diff --git a/lmms_eval/tasks/mmupd/mmivqd_base.yaml b/lmms_eval/tasks/mmupd/mmivqd_base.yaml index 82853fde5..1c8478e2c 100644 --- a/lmms_eval/tasks/mmupd/mmivqd_base.yaml +++ b/lmms_eval/tasks/mmupd/mmivqd_base.yaml @@ -1,7 +1,7 @@ task: "mmivqd_base" test_split: test dataset_name: mmivqd_base -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\n" diff --git a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml b/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml index a024022a7..c6ee19983 100644 --- a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml +++ b/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml @@ -1,7 +1,7 @@ task: "mmivqd_instruction" test_split: test dataset_name: mmivqd_base -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nIf the given image is irrelevant to the question, answer \"F. The image and question are irrelevant.\"." diff --git a/lmms_eval/tasks/mmupd/mmivqd_option.yaml b/lmms_eval/tasks/mmupd/mmivqd_option.yaml index 1363c345c..d1da5e8c7 100644 --- a/lmms_eval/tasks/mmupd/mmivqd_option.yaml +++ b/lmms_eval/tasks/mmupd/mmivqd_option.yaml @@ -1,7 +1,7 @@ task: "mmivqd_option" test_split: test dataset_name: mmivqd_option -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly." diff --git a/lmms_eval/tasks/mmupd/utils.py b/lmms_eval/tasks/mmupd/utils.py index 818e8cb18..a6933835c 100644 --- a/lmms_eval/tasks/mmupd/utils.py +++ b/lmms_eval/tasks/mmupd/utils.py @@ -42,7 +42,7 @@ def mmupd_doc_to_visual(doc): return [Image.open(BytesIO(base64.b64decode(doc["image"])))] -def mmupd_doc_to_text(doc, model_specific_prompt_kwargs=None): +def mmupd_doc_to_text(doc, lmms_eval_specific_kwargs=None): option_candidate = ["A", "B", "C", "D", "E"] options_prompt, options_dict = mmupd_evaluator.create_options_prompt(doc, option_candidate) @@ -61,8 +61,8 @@ def mmupd_doc_to_text(doc, model_specific_prompt_kwargs=None): query_prompt = f"{data['hint']}\n{data['question']}{data['options']}" if pd.notna(data["hint"]) and data["hint"] != "nan" else f"{data['question']}{data['options']}" - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}{model_specific_prompt_kwargs['post_prompt']}" + if lmms_eval_specific_kwargs: + query_prompt = f"{query_prompt}{lmms_eval_specific_kwargs['post_prompt']}" return query_prompt diff --git a/lmms_eval/tasks/mmvet/mmvet.yaml b/lmms_eval/tasks/mmvet/mmvet.yaml index 827856db6..5567ebbca 100755 --- a/lmms_eval/tasks/mmvet/mmvet.yaml +++ b/lmms_eval/tasks/mmvet/mmvet.yaml @@ -21,7 +21,7 @@ metric_list: metadata: version: 0.0 gpt_eval_model_name: "gpt-4-0613" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "First please perform reasoning, and think step by step to provide best answer to the following question: \n\n" post_prompt: "" diff --git a/lmms_eval/tasks/mmvet/utils.py b/lmms_eval/tasks/mmvet/utils.py index 37b15ad73..3b369b073 100755 --- a/lmms_eval/tasks/mmvet/utils.py +++ b/lmms_eval/tasks/mmvet/utils.py @@ -203,11 +203,11 @@ def mmvet_aggregate_results(results): return overall_score -def doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: +def doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: return doc["question"] question = doc["question"] - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/muirbench/muirbench.yaml b/lmms_eval/tasks/muirbench/muirbench.yaml index 896314772..1134898d1 100644 --- a/lmms_eval/tasks/muirbench/muirbench.yaml +++ b/lmms_eval/tasks/muirbench/muirbench.yaml @@ -9,7 +9,7 @@ doc_to_text: !function utils.muir_doc_to_text doc_to_target: !function utils.muir_doc_to_target process_results: !function utils.muir_process_results -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly." diff --git a/lmms_eval/tasks/muirbench/utils.py b/lmms_eval/tasks/muirbench/utils.py index 5364c5bac..8321d7805 100644 --- a/lmms_eval/tasks/muirbench/utils.py +++ b/lmms_eval/tasks/muirbench/utils.py @@ -4,11 +4,11 @@ import pandas as pd -def muir_doc_to_text(doc, model_specific_prompt_kwargs=None): +def muir_doc_to_text(doc, lmms_eval_specific_kwargs=None): question, choices = doc["question"], doc["options"] len_choices = len(choices) - post_prompt = model_specific_prompt_kwargs["post_prompt"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] options = [chr(ord("A") + i) for i in range(len_choices)] choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) return f"{pre_prompt}{question}\n{choices_str}{post_prompt}" diff --git a/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml b/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml index d33c5a50b..2b86155da 100755 --- a/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml +++ b/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml @@ -13,7 +13,7 @@ process_results: !function utils.multidocvqa_process_test_results_for_submission metric_list: - metric: submission aggregation: !function utils.multidocvqa_test_aggregate_results_for_submission -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml b/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml index b4f5238bf..6fe86fb01 100755 --- a/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml +++ b/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml @@ -17,7 +17,7 @@ metric_list: - metric: accuracy aggregation: !function utils.multidocvqa_aggregate_results_accuracy higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/multidocvqa/utils.py b/lmms_eval/tasks/multidocvqa/utils.py index f53da0200..0996cdaee 100755 --- a/lmms_eval/tasks/multidocvqa/utils.py +++ b/lmms_eval/tasks/multidocvqa/utils.py @@ -9,10 +9,10 @@ from loguru import logger as eval_logger -def multidocvqa_doc_to_text(doc, model_specific_prompt_kwargs): +def multidocvqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml index 3ae9260bd..6be6bb224 100644 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml +++ b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml @@ -29,7 +29,7 @@ metric_list: metadata: version: 0.0 gpt_eval_model_name: "gpt-4-0613" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py index b783cb406..bf29636ac 100644 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py +++ b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py @@ -105,11 +105,11 @@ def llava_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") +def llava_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} + pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "") + post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "") return f"{pre_prompt}{doc['question']}{post_prompt}" diff --git a/lmms_eval/tasks/mvbench/_default_template.yaml b/lmms_eval/tasks/mvbench/_default_template.yaml new file mode 100644 index 000000000..89d36dd75 --- /dev/null +++ b/lmms_eval/tasks/mvbench/_default_template.yaml @@ -0,0 +1,23 @@ +dataset_path: OpenGVLab/MVBench +dataset_kwargs: + token: True + cache_dir: mvbench + video: True +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false + +output_type: generate_until +doc_to_visual: !function utils.mvbench_doc_to_visual +doc_to_text: !function utils.mvbench_doc_to_text +doc_to_target: "answer" +# The return value of process_results will be used by metrics +process_results: !function utils.mvbench_process_results +# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results +metric_list: + - metric: mvbench_accuracy + aggregation: !function utils.mvbench_aggregate_results + higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench.yaml b/lmms_eval/tasks/mvbench/mvbench.yaml new file mode 100644 index 000000000..42b42c82f --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench.yaml @@ -0,0 +1,24 @@ +group: mvbench +task: + - mvbench_action_sequence + - mvbench_moving_count + - mvbench_action_prediction + - mvbench_episodic_reasoning + - mvbench_action_antonym + - mvbench_action_count + - mvbench_scene_transition + - mvbench_object_shuffle + - mvbench_object_existence + - mvbench_fine_grained_pose + - mvbench_unexpected_action + - mvbench_moving_direction + - mvbench_state_change + - mvbench_object_interaction + - mvbench_character_order + - mvbench_action_localization + - mvbench_counterfactual_inference + - mvbench_fine_grained_action + - mvbench_moving_attribute + - mvbench_egocentric_navigation + +# ['action_sequence', 'moving_count', 'action_prediction', 'episodic_reasoning', 'action_antonym', 'action_count', 'scene_transition', 'object_shuffle', 'object_existence', 'fine_grained_pose', 'unexpected_action', 'moving_direction', 'state_change', 'object_interaction', 'character_order', 'action_localization', 'counterfactual_inference', 'fine_grained_action', 'moving_attribute', 'egocentric_navigation'] \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_action_antonym.yaml b/lmms_eval/tasks/mvbench/mvbench_action_antonym.yaml new file mode 100644 index 000000000..ca6dbf64d --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_action_antonym.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_action_antonym +dataset_name: action_antonym +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: action_antonym + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_action_count.yaml b/lmms_eval/tasks/mvbench/mvbench_action_count.yaml new file mode 100644 index 000000000..65d41015b --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_action_count.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_action_count +dataset_name: action_count +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: action_count + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_action_localization.yaml b/lmms_eval/tasks/mvbench/mvbench_action_localization.yaml new file mode 100644 index 000000000..c0596162a --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_action_localization.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_action_localization +dataset_name: action_localization +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: action_localization + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_action_prediction.yaml b/lmms_eval/tasks/mvbench/mvbench_action_prediction.yaml new file mode 100644 index 000000000..d41c4f8e4 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_action_prediction.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_action_prediction +dataset_name: action_prediction +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: action_prediction + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_action_sequence.yaml b/lmms_eval/tasks/mvbench/mvbench_action_sequence.yaml new file mode 100644 index 000000000..2d2c20eab --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_action_sequence.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_action_sequence +dataset_name: action_sequence +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: action_sequence + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_character_order.yaml b/lmms_eval/tasks/mvbench/mvbench_character_order.yaml new file mode 100644 index 000000000..d4972cea5 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_character_order.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_character_order +dataset_name: character_order +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: character_order + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_counterfactual_inference.yaml b/lmms_eval/tasks/mvbench/mvbench_counterfactual_inference.yaml new file mode 100644 index 000000000..431c6a2d9 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_counterfactual_inference.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_counterfactual_inference +dataset_name: counterfactual_inference +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: counterfactual_inference + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_egocentric_navigation.yaml b/lmms_eval/tasks/mvbench/mvbench_egocentric_navigation.yaml new file mode 100644 index 000000000..db0b16fb6 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_egocentric_navigation.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_egocentric_navigation +dataset_name: egocentric_navigation +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: egocentric_navigation + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml b/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml new file mode 100644 index 000000000..986c65bcc --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml @@ -0,0 +1,19 @@ +include: _default_template.yaml +task: mvbench_episodic_reasoning +dataset_name: episodic_reasoning +test_split: train +doc_to_visual: !function utils.mvbench_frames_doc_to_visual +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false + +lmms_eval_specific_kwargs: + default: + sub_task: episodic_reasoning + post_prompt: "Answer with the option's letter from the given choices directly." + +metadata: + sample_frames: 32 # for episodic reasoning, it stores the frame images in folder so we need to set the number of frames to sample here. \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_fine_grained_action.yaml b/lmms_eval/tasks/mvbench/mvbench_fine_grained_action.yaml new file mode 100644 index 000000000..10146883f --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_fine_grained_action.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_fine_grained_action +dataset_name: fine_grained_action +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: fine_grained_action + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_fine_grained_pose.yaml b/lmms_eval/tasks/mvbench/mvbench_fine_grained_pose.yaml new file mode 100644 index 000000000..5af1bfe65 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_fine_grained_pose.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_fine_grained_pose +dataset_name: fine_grained_pose +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: fine_grained_pose + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_moving_attribute.yaml b/lmms_eval/tasks/mvbench/mvbench_moving_attribute.yaml new file mode 100644 index 000000000..f5b20a13e --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_moving_attribute.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_moving_attribute +dataset_name: moving_attribute +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: moving_attribute + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_moving_count.yaml b/lmms_eval/tasks/mvbench/mvbench_moving_count.yaml new file mode 100644 index 000000000..0a6b07095 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_moving_count.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_moving_count +dataset_name: moving_count +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: moving_count + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_moving_direction.yaml b/lmms_eval/tasks/mvbench/mvbench_moving_direction.yaml new file mode 100644 index 000000000..bff498f5d --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_moving_direction.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_moving_direction +dataset_name: moving_direction +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: moving_direction + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_object_existence.yaml b/lmms_eval/tasks/mvbench/mvbench_object_existence.yaml new file mode 100644 index 000000000..342ff70f5 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_object_existence.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_object_existence +dataset_name: object_existence +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: object_existence + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_object_interaction.yaml b/lmms_eval/tasks/mvbench/mvbench_object_interaction.yaml new file mode 100644 index 000000000..c884dec95 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_object_interaction.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_object_interaction +dataset_name: object_interaction +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: object_interaction + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_object_shuffle.yaml b/lmms_eval/tasks/mvbench/mvbench_object_shuffle.yaml new file mode 100644 index 000000000..f3119d84a --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_object_shuffle.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_object_shuffle +dataset_name: object_shuffle +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: object_shuffle + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_scene_transition.yaml b/lmms_eval/tasks/mvbench/mvbench_scene_transition.yaml new file mode 100644 index 000000000..ec0b72777 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_scene_transition.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_scene_transition +dataset_name: scene_transition +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: scene_transition + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_state_change.yaml b/lmms_eval/tasks/mvbench/mvbench_state_change.yaml new file mode 100644 index 000000000..5d1195b22 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_state_change.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_state_change +dataset_name: state_change +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: state_change + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/mvbench_unexpected_action.yaml b/lmms_eval/tasks/mvbench/mvbench_unexpected_action.yaml new file mode 100644 index 000000000..e6239ea18 --- /dev/null +++ b/lmms_eval/tasks/mvbench/mvbench_unexpected_action.yaml @@ -0,0 +1,8 @@ +include: _default_template.yaml +task: mvbench_unexpected_action +dataset_name: unexpected_action +test_split: train +lmms_eval_specific_kwargs: + default: + sub_task: unexpected_action + post_prompt: "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mvbench/utils.py b/lmms_eval/tasks/mvbench/utils.py new file mode 100644 index 000000000..5ac0d9a3d --- /dev/null +++ b/lmms_eval/tasks/mvbench/utils.py @@ -0,0 +1,194 @@ +from collections import defaultdict +import os +import datetime +import json +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file +from pathlib import Path +import yaml +import sys, string +from typing import List, Dict, Optional, Union +import re +import PIL +import numpy as np +from loguru import logger as eval_logger + +DATA_LIST = { + "action_sequence": "star/Charades_v1_480/", + "action_prediction": "star/Charades_v1_480/", + "action_antonym": "ssv2_video/", + "fine_grained_action": "Moments_in_Time_Raw/videos/", + "unexpected_action": "FunQA_test/test/", + "object_existence": "clevrer/video_validation/", + "object_interaction": "star/Charades_v1_480/", + "object_shuffle": "perception/videos/", + "moving_direction": "clevrer/video_validation/", + "action_localization": "sta/sta_video/", + "scene_transition": "scene_qa/video/", + "action_count": "perception/videos/", + "moving_count": "clevrer/video_validation/", + "moving_attribute": "clevrer/video_validation/", + "state_change": "perception/videos/", + "fine_grained_pose": "nturgbd/", + "character_order": "perception/videos/", + "egocentric_navigation": "vlnqa/", + "episodic_reasoning": "tvqa/frames_fps3_hq/", + "counterfactual_inference": "clevrer/video_validation/", +} + +hf_home = os.getenv("HF_HOME", "./~/.cache/huggingface") +base_cache_dir = os.path.expanduser(hf_home) + +with open(Path(__file__).parent / "_default_template.yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + +cache_name = yaml.safe_load("".join(safe_data))["dataset_kwargs"]["cache_dir"] + + +def mvbench_doc_to_visual(doc, lmms_eval_specific_kwargs=None): + cache_dir = os.path.join(base_cache_dir, cache_name) + dataset_folder = DATA_LIST[lmms_eval_specific_kwargs["sub_task"]] + video_path = os.path.join(cache_dir, dataset_folder, doc["video"]) + if os.path.exists(video_path): + video_path = video_path + elif os.path.basename(dataset_folder) in ["clevrer", "star"]: + alternative_video_path = os.path.join(cache_dir, "data0613", dataset_folder, doc["video"]) + if os.path.exists(alternative_video_path): + video_path = alternative_video_path + else: + eval_logger.error(f"Video path: {video_path} does not exist, please check.") + else: + eval_logger.error(f"Video path: {video_path} does not exist, please check.") + return [video_path] + + +def mvbench_frames_doc_to_visual(doc, lmms_eval_specific_kwargs=None): + cache_dir = os.path.join(base_cache_dir, cache_name) + dataset_folder = DATA_LIST[lmms_eval_specific_kwargs["sub_task"]] + video_path = os.path.join(cache_dir, dataset_folder, doc["video"]) + if os.path.exists(video_path): + video_path = video_path + elif os.path.basename(dataset_folder) in ["clevrer", "star"]: + alternative_video_path = os.path.join(cache_dir, "data0613", dataset_folder, doc["video"]) + if os.path.exists(alternative_video_path): + video_path = alternative_video_path + else: + eval_logger.error(f"Video path: {video_path} does not exist, please check.") + else: + eval_logger.error(f"Video path: {video_path} does not exist, please check.") + + frame_path_list = [os.path.join(video_path, f) for f in os.listdir(video_path) if f.endswith(".jpg") or f.endswith(".png")] + frame_image_list = [PIL.Image.open(frame_path).convert("RGB") for frame_path in frame_path_list] + return frame_image_list + + +def mvbench_doc_to_text(doc, lmms_eval_specific_kwargs=None): + option_prompt = "" + option_list = doc["candidates"] + option_letters = string.ascii_uppercase + for char_index, option in enumerate(option_list): + option_letter = option_letters[char_index] + option_prompt += f"{option_letter}. {option}\n" + + full_text = doc["question"] + "\n" + option_prompt + lmms_eval_specific_kwargs["post_prompt"] + return full_text + + +def mcq_acc(answer, pred): + periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") + commaStrip = re.compile("(\d)(\,)(\d)") + punct = [";", r"/", "[", "]", '"', "{", "}", "(", ")", "=", "+", "\\", "_", "-", ">", "<", "@", "`", ",", "?", "!"] + + def processPunctuation(inText): + outText = inText + for p in punct: + if (p + " " in inText or " " + p in inText) or (re.search(commaStrip, inText) != None): + outText = outText.replace(p, "") + else: + outText = outText.replace(p, " ") + outText = periodStrip.sub("", outText, re.UNICODE) + return outText + + def process(answer): + option_regex = re.compile(r"^([A-E])\.\s*(.+)$", re.IGNORECASE) + match = option_regex.match(answer.strip()) + + if match: + # If matched, return the option letter in uppercase + return match.group(1).upper() + else: + # If no match, process the answer as before + answer = answer.replace("\n", " ") + answer = answer.replace("\t", " ") + answer = answer.strip() + answer = processPunctuation(answer) + answer = answer.strip("'") + answer = answer.strip('"') + answer = answer.strip(")") + answer = answer.strip("(") + answer = answer.strip().lower() + + # Try to find any single letter (A-E) in the processed answer + letter_match = re.search(r"\b([A-E])\b", answer, re.IGNORECASE) + if letter_match: + return letter_match.group(1).upper() + + return answer + + pred = process(pred) + answer = process(answer) + + if pred == answer: + score = 1 + else: + score = 0 + + return score + + +def mvbench_process_results(doc, results): + """ + Args: + doc: a instance of the eval dataset + results: [pred] + Returns: + a dictionary with key: metric name (in this case mvbench_perception_score), value: metric value + """ + pred = results[0] + + # Calculate the ground truth option letter + option_letters = string.ascii_uppercase + gt_option_letter = None + for i, candidate in enumerate(doc["candidates"]): + if candidate == doc["answer"]: + gt_option_letter = option_letters[i] + break + + # Calculate the score using mcq_acc function + score = mcq_acc(gt_option_letter, pred) + + data_dict = {"pred_answer": pred, "gt_answer": gt_option_letter, "score": score} + + return {"mvbench_accuracy": data_dict} + + +def mvbench_aggregate_results(results): + """ + Args: + results: a list of values returned by process_results + Returns: + A score + """ + total_answered = 0 + total_correct = 0 + for result in results: + if result["pred_answer"] != "": + total_answered += 1 + total_correct += result["score"] + + return 100 * total_correct / total_answered if total_answered > 0 else 0 diff --git a/lmms_eval/tasks/nextqa/nextqa_oe_test.yaml b/lmms_eval/tasks/nextqa/nextqa_oe_test.yaml index 7e5f2c70a..17cae3644 100644 --- a/lmms_eval/tasks/nextqa/nextqa_oe_test.yaml +++ b/lmms_eval/tasks/nextqa/nextqa_oe_test.yaml @@ -11,7 +11,7 @@ metric_list: aggregation: !function utils.nextqa_aggregate_results higher_is_better: true include: _default_template_yaml -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer a question using a short phrase or sentence." diff --git a/lmms_eval/tasks/nextqa/nextqa_oe_val.yaml b/lmms_eval/tasks/nextqa/nextqa_oe_val.yaml index 5511ea3ac..0d9ced097 100644 --- a/lmms_eval/tasks/nextqa/nextqa_oe_val.yaml +++ b/lmms_eval/tasks/nextqa/nextqa_oe_val.yaml @@ -11,7 +11,7 @@ metric_list: aggregation: !function utils.nextqa_aggregate_results higher_is_better: true include: _default_template_yaml -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer a question using a short phrase or sentence." diff --git a/lmms_eval/tasks/nextqa/utils.py b/lmms_eval/tasks/nextqa/utils.py index 6556723b7..8162367f2 100644 --- a/lmms_eval/tasks/nextqa/utils.py +++ b/lmms_eval/tasks/nextqa/utils.py @@ -46,26 +46,26 @@ def nextqa_doc_to_visual(doc): return [get_video(cache_dir, doc["video"])] -def nextqa_doc_to_text(doc, model_specific_prompt_kwargs=None): +def nextqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["question"].strip() - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + if "pre_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["pre_prompt"] != "": + question = f"{lmms_eval_specific_kwargs['pre_prompt']}{question}" + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"] != "": + question = f"{question}{lmms_eval_specific_kwargs['post_prompt']}" return question -def nextqa_doc_to_text_mc(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def nextqa_doc_to_text_mc(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} question = [doc["question"].strip()] for i in range(5): question.append(f"{OPTIONS[i]}. {doc[f'a{i}'].strip()}") question = "\n".join(question) - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + if "pre_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["pre_prompt"] != "": + question = f"{lmms_eval_specific_kwargs['pre_prompt']}{question}" + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"] != "": + question = f"{question}{lmms_eval_specific_kwargs['post_prompt']}" return question diff --git a/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml b/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml index 12fdbbc4d..5953c1f23 100755 --- a/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml +++ b/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml @@ -1,3 +1,3 @@ -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt: "Provide a one-sentence caption for the provided image." \ No newline at end of file diff --git a/lmms_eval/tasks/nocaps/utils.py b/lmms_eval/tasks/nocaps/utils.py index ce6e1d893..cdb853513 100755 --- a/lmms_eval/tasks/nocaps/utils.py +++ b/lmms_eval/tasks/nocaps/utils.py @@ -18,9 +18,9 @@ def nocaps_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def nocaps_doc_to_text(doc, model_specific_prompt_kwargs=None): +def nocaps_doc_to_text(doc, lmms_eval_specific_kwargs=None): # question = "Please carefully observe the image and come up with a caption for the image" - return model_specific_prompt_kwargs["prompt"] + return lmms_eval_specific_kwargs["prompt"] def nocaps_process_result(doc, result): diff --git a/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml b/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml index a69501296..70f595433 100755 --- a/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml +++ b/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml @@ -16,7 +16,7 @@ metric_list: aggregation: !function utils.ok_vqa_aggregate_submissions higher_is_better: true process_results: !function utils.ok_vqa_process_results -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nWhen the provided information is insufficient, respond with 'Unanswerable'.\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/ok_vqa/utils.py b/lmms_eval/tasks/ok_vqa/utils.py index 249e1a0cd..2d9a21391 100755 --- a/lmms_eval/tasks/ok_vqa/utils.py +++ b/lmms_eval/tasks/ok_vqa/utils.py @@ -48,16 +48,16 @@ def ok_vqa_process_results(doc, result): } -def ok_vqa_doc_to_text(doc, model_specific_prompt_kwargs=None): +def ok_vqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["question"] - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/perceptiontest/test/_default_template_yaml b/lmms_eval/tasks/perceptiontest/test/_default_template_yaml index 39ea0aa42..8a35beb55 100644 --- a/lmms_eval/tasks/perceptiontest/test/_default_template_yaml +++ b/lmms_eval/tasks/perceptiontest/test/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: perceptiontest -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/perceptiontest/test/utils.py b/lmms_eval/tasks/perceptiontest/test/utils.py index 522a6a041..062947767 100755 --- a/lmms_eval/tasks/perceptiontest/test/utils.py +++ b/lmms_eval/tasks/perceptiontest/test/utils.py @@ -45,15 +45,15 @@ def perceptiontest_doc_to_visual(doc): # This is the place where you format your question -def perceptiontest_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def perceptiontest_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["question"] if "options" in doc: diff --git a/lmms_eval/tasks/perceptiontest/val/_default_template_yaml b/lmms_eval/tasks/perceptiontest/val/_default_template_yaml index 82dd37caa..c586beb97 100644 --- a/lmms_eval/tasks/perceptiontest/val/_default_template_yaml +++ b/lmms_eval/tasks/perceptiontest/val/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: perceptiontest_val -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/perceptiontest/val/utils.py b/lmms_eval/tasks/perceptiontest/val/utils.py index 63156fe28..3caa73a83 100755 --- a/lmms_eval/tasks/perceptiontest/val/utils.py +++ b/lmms_eval/tasks/perceptiontest/val/utils.py @@ -45,15 +45,15 @@ def perceptiontest_val_doc_to_visual(doc): # This is the place where you format your question -def perceptiontest_val_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def perceptiontest_val_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["question"] if "options" in doc: diff --git a/lmms_eval/tasks/qbench/abench_dev.yaml b/lmms_eval/tasks/qbench/abench_dev.yaml index 8b44b5f7d..fb8140e8f 100644 --- a/lmms_eval/tasks/qbench/abench_dev.yaml +++ b/lmms_eval/tasks/qbench/abench_dev.yaml @@ -15,7 +15,7 @@ metric_list: aggregation: !function utils.a_bench_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "Answer with the option's letter from the given choices directly.\n" diff --git a/lmms_eval/tasks/qbench/qbench2_dev.yaml b/lmms_eval/tasks/qbench/qbench2_dev.yaml index 7412cb25c..ae8dc5c7f 100644 --- a/lmms_eval/tasks/qbench/qbench2_dev.yaml +++ b/lmms_eval/tasks/qbench/qbench2_dev.yaml @@ -15,7 +15,7 @@ metric_list: aggregation: !function utils.q_bench_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "Answer with the option's letter from the given choices directly.\n" diff --git a/lmms_eval/tasks/qbench/qbench_dev.yaml b/lmms_eval/tasks/qbench/qbench_dev.yaml index be901333d..97d9df20d 100644 --- a/lmms_eval/tasks/qbench/qbench_dev.yaml +++ b/lmms_eval/tasks/qbench/qbench_dev.yaml @@ -15,7 +15,7 @@ metric_list: aggregation: !function utils.q_bench_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "Answer with the option's letter from the given choices directly.\n" diff --git a/lmms_eval/tasks/qbench/utils.py b/lmms_eval/tasks/qbench/utils.py index a56415702..cc3ce1201 100644 --- a/lmms_eval/tasks/qbench/utils.py +++ b/lmms_eval/tasks/qbench/utils.py @@ -4,7 +4,7 @@ from lmms_eval.tasks._task_utils.file_utils import generate_submission_file -def q_bench_doc_to_text(doc, model_specific_prompt_kwargs): +def q_bench_doc_to_text(doc, lmms_eval_specific_kwargs): candidates = [] for i in range(4): candidate = doc.get(f"option{i}") @@ -12,8 +12,8 @@ def q_bench_doc_to_text(doc, model_specific_prompt_kwargs): candidates.append(candidate) question = doc["question"] + "\n" + "\n".join([". ".join([chr(ord("A") + i), candidate]) for i, candidate in enumerate(candidates)]) - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}\n{post_prompt}" diff --git a/lmms_eval/tasks/realworldqa/realworldqa.yaml b/lmms_eval/tasks/realworldqa/realworldqa.yaml index 4b4149672..cf407b2f2 100644 --- a/lmms_eval/tasks/realworldqa/realworldqa.yaml +++ b/lmms_eval/tasks/realworldqa/realworldqa.yaml @@ -32,7 +32,7 @@ metric_list: ignore_case: true ignore_punctuation: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/realworldqa/utils.py b/lmms_eval/tasks/realworldqa/utils.py index 9ed645ed1..ecde1f4b7 100644 --- a/lmms_eval/tasks/realworldqa/utils.py +++ b/lmms_eval/tasks/realworldqa/utils.py @@ -9,17 +9,17 @@ def realworldqa_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def realworldqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def realworldqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" question = doc["question"].strip() - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"]: + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"]: question = question.replace(REPLACE_PROMPT, "") - post_prompt = model_specific_prompt_kwargs["post_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/scienceqa/scienceqa.yaml b/lmms_eval/tasks/scienceqa/scienceqa.yaml index 8190e3e90..f34b19fa2 100755 --- a/lmms_eval/tasks/scienceqa/scienceqa.yaml +++ b/lmms_eval/tasks/scienceqa/scienceqa.yaml @@ -22,7 +22,7 @@ process_results: !function utils.sqa_process_results metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: format: default pre_prompt: "" diff --git a/lmms_eval/tasks/scienceqa/scienceqa_img.yaml b/lmms_eval/tasks/scienceqa/scienceqa_img.yaml index b6c7dcc6f..7faa77285 100755 --- a/lmms_eval/tasks/scienceqa/scienceqa_img.yaml +++ b/lmms_eval/tasks/scienceqa/scienceqa_img.yaml @@ -22,7 +22,7 @@ process_results: !function utils.sqa_process_results metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: format: default pre_prompt: "" diff --git a/lmms_eval/tasks/scienceqa/utils.py b/lmms_eval/tasks/scienceqa/utils.py index eed6b26ab..49587b5dd 100755 --- a/lmms_eval/tasks/scienceqa/utils.py +++ b/lmms_eval/tasks/scienceqa/utils.py @@ -1,22 +1,22 @@ -def sqa_doc_to_text(doc, model_specific_prompt_kwargs=None): +def sqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): context, question, choices = doc["hint"], doc["question"], doc["choices"] len_choices = len(choices) options = [chr(ord("A") + i) for i in range(len_choices)] choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) - if model_specific_prompt_kwargs["format"] == "default": + if lmms_eval_specific_kwargs["format"] == "default": if context: context = f"Context: {context}\n" - post_prompt = model_specific_prompt_kwargs["post_prompt"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] return f"{pre_prompt}{context}{question}\n{choices_str}{post_prompt}" - elif model_specific_prompt_kwargs["format"] == "qwen_vl": + elif lmms_eval_specific_kwargs["format"] == "qwen_vl": prompt = "Context: {}\nQuestion: {}\nOptions: {}\nAnswer:" context = context if context else "N/A" prompt = prompt.format(context, question, choices_str) return prompt else: - raise ValueError(f"Unknown prompt format: {model_specific_prompt_kwargs}") + raise ValueError(f"Unknown prompt format: {lmms_eval_specific_kwargs}") def sqa_doc_to_visual(doc): diff --git a/lmms_eval/tasks/seedbench_2/seedbench_2.yaml b/lmms_eval/tasks/seedbench_2/seedbench_2.yaml index c61e4104b..9f874da35 100755 --- a/lmms_eval/tasks/seedbench_2/seedbench_2.yaml +++ b/lmms_eval/tasks/seedbench_2/seedbench_2.yaml @@ -40,10 +40,13 @@ metric_list: metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: llava : img_token : post_prompt : "Answer with the option's letter from the given choices directly." + llava_onevision : + img_token : + post_prompt : "Answer with the option's letter from the given choices directly." gpt4V : img_token : post_prompt : "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/stvqa/stvqa.yaml b/lmms_eval/tasks/stvqa/stvqa.yaml index 42ecaacad..9d049b4a9 100755 --- a/lmms_eval/tasks/stvqa/stvqa.yaml +++ b/lmms_eval/tasks/stvqa/stvqa.yaml @@ -13,7 +13,7 @@ process_results: !function utils.stvqa_process_results metric_list: - metric: submission aggregation: !function utils.stvqa_aggregate_submissions -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/stvqa/utils.py b/lmms_eval/tasks/stvqa/utils.py index ff033f1f9..0edd7d72f 100755 --- a/lmms_eval/tasks/stvqa/utils.py +++ b/lmms_eval/tasks/stvqa/utils.py @@ -4,10 +4,10 @@ from lmms_eval.tasks._task_utils.file_utils import generate_submission_file -def stvqa_doc_to_text(doc, model_specific_prompt_kwargs): +def stvqa_doc_to_text(doc, lmms_eval_specific_kwargs): question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/tempcompass/_default_template_yaml b/lmms_eval/tasks/tempcompass/_default_template_yaml index 404701d30..9b5087d4d 100644 --- a/lmms_eval/tasks/tempcompass/_default_template_yaml +++ b/lmms_eval/tasks/tempcompass/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: tempcompass -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: { diff --git a/lmms_eval/tasks/tempcompass/utils.py b/lmms_eval/tasks/tempcompass/utils.py index dbe4f8311..cabc67da9 100644 --- a/lmms_eval/tasks/tempcompass/utils.py +++ b/lmms_eval/tasks/tempcompass/utils.py @@ -64,57 +64,57 @@ def tempcompass_doc_to_visual(doc): # This is the place where you format your question -def tempcompass_doc_to_text_multi_choice(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def tempcompass_doc_to_text_multi_choice(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"]["multi-choice"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"]["multi-choice"] question = doc["question"] return f"{pre_prompt}{question}{post_prompt}" -def tempcompass_doc_to_text_yes_no(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def tempcompass_doc_to_text_yes_no(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"]["yes_no"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"]["yes_no"] question = doc["question"] return f"{pre_prompt}{question}{post_prompt}" -def tempcompass_doc_to_text_caption_matching(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def tempcompass_doc_to_text_caption_matching(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"]["caption_matching"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"]["caption_matching"] question = doc["question"] return f"{pre_prompt}{question}{post_prompt}" -def tempcompass_doc_to_text_captioning(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def tempcompass_doc_to_text_captioning(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"]["captioning"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"]["captioning"] question = doc["question"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml b/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml index bb71de494..cfed932f7 100755 --- a/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml +++ b/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml @@ -1,3 +1,3 @@ -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt: Provide a one-sentence caption for the provided image. \ No newline at end of file diff --git a/lmms_eval/tasks/textcaps/utils.py b/lmms_eval/tasks/textcaps/utils.py index d21a92fdd..bc8b266f7 100755 --- a/lmms_eval/tasks/textcaps/utils.py +++ b/lmms_eval/tasks/textcaps/utils.py @@ -17,8 +17,8 @@ def textcaps_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def textcaps_doc_to_text(doc, model_specific_prompt_kwargs=None): - return model_specific_prompt_kwargs["prompt"] +def textcaps_doc_to_text(doc, lmms_eval_specific_kwargs=None): + return lmms_eval_specific_kwargs["prompt"] def textcaps_process_result(doc, result): diff --git a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml b/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml index 282b401b6..806edecae 100755 --- a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml +++ b/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml @@ -7,7 +7,7 @@ generation_kwargs: until: - "ASSISTANT:" process_results: !function utils.textvqa_process_results -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/textvqa/utils.py b/lmms_eval/tasks/textvqa/utils.py index 0c8de47d1..c9ea6d22c 100755 --- a/lmms_eval/tasks/textvqa/utils.py +++ b/lmms_eval/tasks/textvqa/utils.py @@ -45,16 +45,16 @@ def textvqa_process_results(doc, result): } -def textvqa_doc_to_text(doc, model_specific_prompt_kwargs=None): +def textvqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): pre_prompt = "" post_post = "" ocr_ref = "" - if model_specific_prompt_kwargs: - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - if "ocr" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["ocr"]: + if lmms_eval_specific_kwargs: + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] + if "ocr" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["ocr"]: ocr_ref = f"\nReference OCR token: {', '.join(doc['ocr_tokens'])}" return f"{pre_prompt}{doc['question'].capitalize()}{ocr_ref}{post_prompt}" diff --git a/lmms_eval/tasks/vatex/utils.py b/lmms_eval/tasks/vatex/utils.py index cdb62806d..ea3c93a0a 100755 --- a/lmms_eval/tasks/vatex/utils.py +++ b/lmms_eval/tasks/vatex/utils.py @@ -77,14 +77,14 @@ def vatex_test_doc_to_visual(doc): return [video_path] -def vatex_ZH_doc_to_text(doc, model_specific_prompt_kwargs=None): +def vatex_ZH_doc_to_text(doc, lmms_eval_specific_kwargs=None): few_shot_prompt = """[视频1] 输出:一个穿黑运动服、戴红色头盔的男人正在攀登雪山。\n[视频2] 输出:一个戴着耳机男人在电脑面前模拟打架子鼓。\n[视频3] 输出:一个穿黑色短袖的男子的男子，双手十指交叉放在胸前，肘部放在面前的桌子上，桌子上有一台电脑，不一会儿，男子半个手臂都放在了桌子上。\n[视频4] 输出:一位女士在她的手上涂抹少量的面霜，并且在她的眼睛下涂抹。\n""" - return model_specific_prompt_kwargs["prompt"] + "\n" + few_shot_prompt + return lmms_eval_specific_kwargs["prompt"] + "\n" + few_shot_prompt -def vatex_test_doc_to_text(doc, model_specific_prompt_kwargs=None): +def vatex_test_doc_to_text(doc, lmms_eval_specific_kwargs=None): few_shot_prompt = """[video1] output: A man picks up a can of shoe paste, a towel, and brush from a table.\n[video2] output: A person places the frying pan on the stove and then another person flips over the food that is in it.\n[video3] output: A woman describes and demonstrates how to create a colorful cross stitch design.\n[video4] output: A little girl uses the grass in her yard as well as a blue mat to do flips.\n""" - return model_specific_prompt_kwargs["prompt"] + "\n" + few_shot_prompt + return lmms_eval_specific_kwargs["prompt"] + "\n" + few_shot_prompt def vatex_process_result(doc, result): diff --git a/lmms_eval/tasks/vatex/vatex_test.yaml b/lmms_eval/tasks/vatex/vatex_test.yaml index 0087e9771..dd991945e 100644 --- a/lmms_eval/tasks/vatex/vatex_test.yaml +++ b/lmms_eval/tasks/vatex/vatex_test.yaml @@ -32,7 +32,7 @@ dataset_kwargs: cache_dir: vatex_test # From_YouTube: True -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt: Provide a brief single-sentence caption for the last video below. Do not give any reasoning, just the caption. You must follow the captioning style of the preceding videos. Do not start your response with "Output:", just provide the caption. gemini_api: diff --git a/lmms_eval/tasks/vatex/vatex_val_zh.yaml b/lmms_eval/tasks/vatex/vatex_val_zh.yaml index cc2a866b2..598145a80 100755 --- a/lmms_eval/tasks/vatex/vatex_val_zh.yaml +++ b/lmms_eval/tasks/vatex/vatex_val_zh.yaml @@ -12,7 +12,7 @@ generation_kwargs: top_p: 1.0 num_beams: 1 do_sample: false -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt: 请为提供的视频提供简短的描述。不要给出任何理由，只提供描述。您必须沿用前面视频的描述样式。不需要以 "输出"开头，只需提供描述即可. diff --git a/lmms_eval/tasks/vcr_wiki/utils.py b/lmms_eval/tasks/vcr_wiki/utils.py index 5519c0787..9ffde90c4 100644 --- a/lmms_eval/tasks/vcr_wiki/utils.py +++ b/lmms_eval/tasks/vcr_wiki/utils.py @@ -73,11 +73,11 @@ def vcr_doc_to_visual(doc): return [doc["stacked_image"].convert("RGB")] -def vcr_doc_to_text(doc, model_specific_prompt_kwargs=None): - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] +def vcr_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{post_prompt}" diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy.yaml index 9ce88299d..2fa1c6022 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "What is the covered texts in the image? Please restore the covered texts without outputting the explanations." \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_100.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_100.yaml index 37f0130b2..3e99cec62 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_100.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_100.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "What is the covered texts in the image? Please restore the covered texts without outputting the explanations." \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_500.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_500.yaml index 0ccc1ed0c..c9ff384d1 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_500.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_easy_500.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "What is the covered texts in the image? Please restore the covered texts without outputting the explanations." \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard.yaml index 8c001e4e5..a99eca67e 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "What is the covered texts in the image? Please restore the covered texts without outputting the explanations." \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_100.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_100.yaml index 16c79b15f..4b00a2fcf 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_100.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_100.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "What is the covered texts in the image? Please restore the covered texts without outputting the explanations." \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_500.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_500.yaml index 656168947..f8788e666 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_500.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_en_hard_500.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "What is the covered texts in the image? Please restore the covered texts without outputting the explanations." \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy.yaml index a07cf2306..32cf502a8 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "图像中被覆盖的文本是什么？请在不输出解释的情况下还原被覆盖的文本。" \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_100.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_100.yaml index fb41b668d..0c317556a 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_100.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_100.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "图像中被覆盖的文本是什么？请在不输出解释的情况下还原被覆盖的文本。" \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_500.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_500.yaml index e3654aed7..0407f9853 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_500.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_easy_500.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "图像中被覆盖的文本是什么？请在不输出解释的情况下还原被覆盖的文本。" \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard.yaml index 0401b3bdd..38cc516e2 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "图像中被覆盖的文本是什么？请在不输出解释的情况下还原被覆盖的文本。" \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_100.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_100.yaml index f6512e293..375dff206 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_100.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_100.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "图像中被覆盖的文本是什么？请在不输出解释的情况下还原被覆盖的文本。" \ No newline at end of file diff --git a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_500.yaml b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_500.yaml index a36bb328e..02b34f537 100644 --- a/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_500.yaml +++ b/lmms_eval/tasks/vcr_wiki/vcr_wiki_zh_hard_500.yaml @@ -10,7 +10,7 @@ metric_list: - metric: exact_match aggregation: !function utils.vcr_aggregate_exact_match higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "图像中被覆盖的文本是什么？请在不输出解释的情况下还原被覆盖的文本。" \ No newline at end of file diff --git a/lmms_eval/tasks/vibe_eval/utils.py b/lmms_eval/tasks/vibe_eval/utils.py index 1ba1cdb71..09b0c58a2 100644 --- a/lmms_eval/tasks/vibe_eval/utils.py +++ b/lmms_eval/tasks/vibe_eval/utils.py @@ -139,12 +139,12 @@ def vibe_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def vibe_doc_to_text(doc, model_specific_prompt_kwargs=None): +def vibe_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["prompt"].strip() - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + if "pre_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["pre_prompt"] != "": + question = f"{lmms_eval_specific_kwargs['pre_prompt']}{question}" + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"] != "": + question = f"{question}{lmms_eval_specific_kwargs['post_prompt']}" return question diff --git a/lmms_eval/tasks/vibe_eval/vibe_eval.yaml b/lmms_eval/tasks/vibe_eval/vibe_eval.yaml index ed7f31c64..a932b5779 100644 --- a/lmms_eval/tasks/vibe_eval/vibe_eval.yaml +++ b/lmms_eval/tasks/vibe_eval/vibe_eval.yaml @@ -29,7 +29,7 @@ metric_list: metadata: evaluator: "reka-core-text" -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/video_detail_description/_default_template_yaml b/lmms_eval/tasks/video_detail_description/_default_template_yaml index 914bf8ee3..dd0948c11 100644 --- a/lmms_eval/tasks/video_detail_description/_default_template_yaml +++ b/lmms_eval/tasks/video_detail_description/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: videochatgpt -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/video_detail_description/utils.py b/lmms_eval/tasks/video_detail_description/utils.py index 536776fb2..f8dae7fb4 100755 --- a/lmms_eval/tasks/video_detail_description/utils.py +++ b/lmms_eval/tasks/video_detail_description/utils.py @@ -64,15 +64,15 @@ def video_detail_description_doc_to_visual(doc): # format the question -def video_detail_description_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def video_detail_description_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["question"] return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/videochatgpt/_default_template_yaml b/lmms_eval/tasks/videochatgpt/_default_template_yaml index 816297ccb..6d3573af8 100644 --- a/lmms_eval/tasks/videochatgpt/_default_template_yaml +++ b/lmms_eval/tasks/videochatgpt/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: videochatgpt -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/videochatgpt/utils.py b/lmms_eval/tasks/videochatgpt/utils.py index e59f2dbfc..d700ee786 100755 --- a/lmms_eval/tasks/videochatgpt/utils.py +++ b/lmms_eval/tasks/videochatgpt/utils.py @@ -66,15 +66,15 @@ def videochatgpt_doc_to_visual(doc): # format the question -def videochatgpt_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def videochatgpt_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["question"] @@ -82,15 +82,15 @@ def videochatgpt_doc_to_text(doc, model_specific_prompt_kwargs=None): # format the question for consistency -def videochatgpt_doc_to_text_consistency(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def videochatgpt_doc_to_text_consistency(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] if doc["question_1"] != "None": question = doc["question_1"] diff --git a/lmms_eval/tasks/videomme/utils.py b/lmms_eval/tasks/videomme/utils.py index c9ec7aa6e..98a9cbff1 100644 --- a/lmms_eval/tasks/videomme/utils.py +++ b/lmms_eval/tasks/videomme/utils.py @@ -134,41 +134,45 @@ def extract_subtitles(video_path, subtitle_path): def parse_subtitle_time(time_str): - h, m, s_ms = time_str.split(':') - s, ms = s_ms.split(',') + h, m, s_ms = time_str.split(":") + s, ms = s_ms.split(",") return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000 + def load_subtitles(subtitle_path): subtitles = {} - with open(subtitle_path, 'r', encoding='utf-8') as file: - content = file.read().split('\n\n') + with open(subtitle_path, "r", encoding="utf-8") as file: + content = file.read().split("\n\n") for section in content: if section.strip(): - lines = section.split('\n') + lines = section.split("\n") if len(lines) >= 3: - time_range = lines[1].split(' --> ') + time_range = lines[1].split(" --> ") start_time = parse_subtitle_time(time_range[0]) end_time = parse_subtitle_time(time_range[1]) - text = ' '.join(line for line in lines[2:]) + text = " ".join(line for line in lines[2:]) subtitles[(start_time, end_time)] = text return subtitles + def convert_time_to_frame(time_in_seconds, fps): return int(time_in_seconds * fps) + def extract_subtitles(video_path, subtitle_path): video = cv2.VideoCapture(video_path) fps = video.get(cv2.CAP_PROP_FPS) - total_frame=int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + total_frame = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) subtitles = load_subtitles(subtitle_path) - + subtitle_frames = [] for (start_time, end_time), text in subtitles.items(): start_frame = convert_time_to_frame(start_time, fps) end_frame = convert_time_to_frame(end_time, fps) subtitle_frames.append((start_frame, end_frame, text)) - return subtitle_frames,total_frame + return subtitle_frames, total_frame + def videomme_doc_to_visual(doc): cache_dir = os.path.join(base_cache_dir, cache_name) @@ -185,15 +189,17 @@ def videomme_doc_to_visual(doc): return [video_path] -def videomme_doc_to_text(doc, model_specific_prompt_kwargs=None): - option_prompt="Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." +def videomme_doc_to_text(doc, lmms_eval_specific_kwargs=None): + option_prompt = "Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." question = doc["question"] option = str(doc["options"]) question = question + "\n" + option - full_prompt=option_prompt+"\n"+question+"\n"+"The best answer is:" + full_prompt = option_prompt + "\n" + question + "\n" + "The best answer is:" return full_prompt + + # Frames + Subs -# This video's subtitles are listed below: +# This video's subtitles are listed below: # 【subtitles】 # Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option. @@ -204,53 +210,54 @@ def videomme_doc_to_text(doc, model_specific_prompt_kwargs=None): # 【question】 # The best answer is: -def videomme_doc_to_text_subtitle(doc, model_specific_prompt_kwargs=None): + +def videomme_doc_to_text_subtitle(doc, lmms_eval_specific_kwargs=None): cache_dir = os.path.join(base_cache_dir, cache_name) video_path = doc["videoID"] + ".mp4" - subtitle_path=os.path.join(cache_dir,"subtitle",doc["videoID"]+".srt") + subtitle_path = os.path.join(cache_dir, "subtitle", doc["videoID"] + ".srt") video_path = os.path.join(cache_dir, video_path) - if os.path.exists(subtitle_path): #Denote have subtitle - subtitle=open(subtitle_path).readlines() + if os.path.exists(subtitle_path): # Denote have subtitle + subtitle = open(subtitle_path).readlines() else: - subtitle="" - subtitles_prompt="This video's subtitles are listed below: \n" - if subtitle=="": - subtitle="No subtitles available" + subtitle = "" + subtitles_prompt = "This video's subtitles are listed below: \n" + if subtitle == "": + subtitle = "No subtitles available" else: - if "gemini_api_flag" in model_specific_prompt_kwargs: #specific for gemini_api - if model_specific_prompt_kwargs['gemini_api_flag']=="full subtitle": - textlist=[] + if "gemini_api_flag" in lmms_eval_specific_kwargs: # specific for gemini_api + if lmms_eval_specific_kwargs["gemini_api_flag"] == "full subtitle": + textlist = [] for ele in subtitle: pattern = r'(.*?)' matches = re.findall(pattern, ele) if matches: textlist.append(matches[0]) - subtitle_text="\n".join(textlist) + subtitle_text = "\n".join(textlist) else: - if "frame_num" in model_specific_prompt_kwargs: - frame_num=model_specific_prompt_kwargs['frame_num'] - subtitle_by_frame,total_frame=extract_subtitles(video_path,subtitle_path) + if "frame_num" in lmms_eval_specific_kwargs: + frame_num = lmms_eval_specific_kwargs["frame_num"] + subtitle_by_frame, total_frame = extract_subtitles(video_path, subtitle_path) uniform_sampled_frames = np.linspace(0, total_frame - 1, frame_num, dtype=int).tolist() - - subtitle_by_frame_idx=[] + + subtitle_by_frame_idx = [] for frame_idx in uniform_sampled_frames: - for idx,title in enumerate(subtitle_by_frame): - if frame_idx=title[0]: + for idx, title in enumerate(subtitle_by_frame): + if frame_idx < title[1] and frame_idx >= title[0]: subtitle_by_frame_idx.append(idx) - subtitle_by_frame_idx=list(set(subtitle_by_frame_idx)) + subtitle_by_frame_idx = list(set(subtitle_by_frame_idx)) - textlist=[] + textlist = [] for idx in subtitle_by_frame_idx: pattern = r'(.*?)' - raw_text=re.findall(pattern, subtitle_by_frame[idx][2]) + raw_text = re.findall(pattern, subtitle_by_frame[idx][2]) try: textlist.append(raw_text[0]) except: continue - subtitle_text="\n".join(textlist) - subtitle=subtitle_text + subtitle_text = "\n".join(textlist) + subtitle = subtitle_text - option_prompt="Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." + option_prompt = "Select the best answer to the following multiple-choice question based on the video and the subtitles. Respond with only the letter (A, B, C, or D) of the correct option." question = doc["question"] option = str(doc["options"]) # option = "\n".join([f"{opt}" for i, opt in enumerate(doc["options"])]) @@ -272,7 +279,7 @@ def videomme_doc_to_text_subtitle(doc, model_specific_prompt_kwargs=None): # The best answer is: -def videomme_doc_to_text_subtitle(doc, model_specific_prompt_kwargs=None): +def videomme_doc_to_text_subtitle(doc, lmms_eval_specific_kwargs=None): cache_dir = os.path.join(base_cache_dir, cache_name) video_path = doc["videoID"] + ".mp4" subtitle_path = os.path.join(cache_dir, "subtitle", doc["videoID"] + ".srt") @@ -285,8 +292,8 @@ def videomme_doc_to_text_subtitle(doc, model_specific_prompt_kwargs=None): if subtitle == "": subtitle = "No subtitles available" else: - if "gemini_api_flag" in model_specific_prompt_kwargs: # specific for gemini_api - if model_specific_prompt_kwargs["gemini_api_flag"] == "full subtitle": + if "gemini_api_flag" in lmms_eval_specific_kwargs: # specific for gemini_api + if lmms_eval_specific_kwargs["gemini_api_flag"] == "full subtitle": textlist = [] for ele in subtitle: pattern = r'(.*?)' @@ -295,8 +302,8 @@ def videomme_doc_to_text_subtitle(doc, model_specific_prompt_kwargs=None): textlist.append(matches[0]) subtitle_text = "\n".join(textlist) else: - if "frame_num" in model_specific_prompt_kwargs: - frame_num = model_specific_prompt_kwargs["frame_num"] + if "frame_num" in lmms_eval_specific_kwargs: + frame_num = lmms_eval_specific_kwargs["frame_num"] subtitle_by_frame, total_frame = extract_subtitles(video_path, subtitle_path) uniform_sampled_frames = np.linspace(0, total_frame - 1, frame_num, dtype=int).tolist() diff --git a/lmms_eval/tasks/videomme/videomme.yaml b/lmms_eval/tasks/videomme/videomme.yaml index 7689da737..cc40dd0cc 100644 --- a/lmms_eval/tasks/videomme/videomme.yaml +++ b/lmms_eval/tasks/videomme/videomme.yaml @@ -23,7 +23,7 @@ metric_list: - metric: videomme_percetion_score aggregation: !function utils.videomme_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question with A, B, C, or D." diff --git a/lmms_eval/tasks/videomme/videomme_w_subtitle.yaml b/lmms_eval/tasks/videomme/videomme_w_subtitle.yaml index cfa5f0704..938bb2f22 100644 --- a/lmms_eval/tasks/videomme/videomme_w_subtitle.yaml +++ b/lmms_eval/tasks/videomme/videomme_w_subtitle.yaml @@ -23,7 +23,7 @@ metric_list: - metric: videomme_percetion_score aggregation: !function utils.videomme_aggregate_results higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: frame_num: 32 pre_prompt: "" diff --git a/lmms_eval/tasks/vitatecs/_default_template_yaml b/lmms_eval/tasks/vitatecs/_default_template_yaml index e93ce7d5e..67547982f 100644 --- a/lmms_eval/tasks/vitatecs/_default_template_yaml +++ b/lmms_eval/tasks/vitatecs/_default_template_yaml @@ -3,7 +3,7 @@ dataset_kwargs: token: True video: True cache_dir: vitatecs -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nPlease response with a single letter (A or B):" \ No newline at end of file diff --git a/lmms_eval/tasks/vitatecs/utils.py b/lmms_eval/tasks/vitatecs/utils.py index b2adcaa96..af2b09abf 100644 --- a/lmms_eval/tasks/vitatecs/utils.py +++ b/lmms_eval/tasks/vitatecs/utils.py @@ -62,15 +62,15 @@ def vitatecs_doc_to_visual(doc): # This is the place where you format your question -def vitatecs_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def vitatecs_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question, _, _ = format_question_and_answer(doc) return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml b/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml index b7dd4137a..248c763a4 100755 --- a/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml +++ b/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml @@ -8,7 +8,7 @@ generation_kwargs: - "ASSISTANT:" metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nWhen the provided information is insufficient, respond with 'Unanswerable'.\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/vizwiz_vqa/utils.py b/lmms_eval/tasks/vizwiz_vqa/utils.py index 9be1f574c..b17cf8d1d 100755 --- a/lmms_eval/tasks/vizwiz_vqa/utils.py +++ b/lmms_eval/tasks/vizwiz_vqa/utils.py @@ -48,15 +48,15 @@ def vizwiz_vqa_process_results(doc, result): } -def vizwiz_vqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def vizwiz_vqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] text = f"{pre_prompt}{doc['question'].capitalize()}{post_prompt}" return text diff --git a/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml b/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml index d3ce20f77..ef0a7c8b7 100755 --- a/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml +++ b/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml @@ -9,7 +9,7 @@ generation_kwargs: max_new_tokens: 16 metadata: - version: 0.0 -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer the question using a single word or phrase." \ No newline at end of file diff --git a/lmms_eval/tasks/vqav2/utils.py b/lmms_eval/tasks/vqav2/utils.py index f70a3965f..71518e0ca 100755 --- a/lmms_eval/tasks/vqav2/utils.py +++ b/lmms_eval/tasks/vqav2/utils.py @@ -68,15 +68,15 @@ def vqav2_process_results_val(doc, result): } -def vqav2_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def vqav2_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] return f"{pre_prompt}{doc['question']}{post_prompt}" diff --git a/lmms_eval/tasks/wild_vision_bench/utils.py b/lmms_eval/tasks/wild_vision_bench/utils.py index f106d0047..3913a1516 100644 --- a/lmms_eval/tasks/wild_vision_bench/utils.py +++ b/lmms_eval/tasks/wild_vision_bench/utils.py @@ -132,12 +132,12 @@ def wild_vision_doc_to_visual(doc): return [doc["image"].convert("RGB")] -def wild_vision_doc_to_text(doc, model_specific_prompt_kwargs=None): +def wild_vision_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["instruction"].strip() - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" + if "pre_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["pre_prompt"] != "": + question = f"{lmms_eval_specific_kwargs['pre_prompt']}{question}" + if "post_prompt" in lmms_eval_specific_kwargs and lmms_eval_specific_kwargs["post_prompt"] != "": + question = f"{question}{lmms_eval_specific_kwargs['post_prompt']}" return question diff --git a/lmms_eval/tasks/wild_vision_bench/wild_vision_bench0617.yaml b/lmms_eval/tasks/wild_vision_bench/wild_vision_bench0617.yaml index 93576768f..88ba56903 100644 --- a/lmms_eval/tasks/wild_vision_bench/wild_vision_bench0617.yaml +++ b/lmms_eval/tasks/wild_vision_bench/wild_vision_bench0617.yaml @@ -3,7 +3,7 @@ dataset_name: release_bench_0617_with_modelresponse test_split: test500 output_type: generate_until include: _default_template_yaml -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/worldqa/utils.py b/lmms_eval/tasks/worldqa/utils.py index bd9b89ef0..42d4059fb 100755 --- a/lmms_eval/tasks/worldqa/utils.py +++ b/lmms_eval/tasks/worldqa/utils.py @@ -132,15 +132,15 @@ def worldqa_doc_to_visual(doc): # This is the place where you format your question -def worldqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} +def worldqa_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs is None: + lmms_eval_specific_kwargs = {} pre_prompt = "" post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] + if "pre_prompt" in lmms_eval_specific_kwargs: + pre_prompt = lmms_eval_specific_kwargs["pre_prompt"] + if "post_prompt" in lmms_eval_specific_kwargs: + post_prompt = lmms_eval_specific_kwargs["post_prompt"] question = doc["question"] if "option" in doc: diff --git a/lmms_eval/tasks/worldqa/worldqa_generation.yaml b/lmms_eval/tasks/worldqa/worldqa_generation.yaml index f5ed41e9f..78db45bce 100755 --- a/lmms_eval/tasks/worldqa/worldqa_generation.yaml +++ b/lmms_eval/tasks/worldqa/worldqa_generation.yaml @@ -13,7 +13,7 @@ metric_list: - metric: gpt_eval aggregation: !function utils.worldq_gen_gpt_eval higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/worldqa/worldqa_mc.yaml b/lmms_eval/tasks/worldqa/worldqa_mc.yaml index 821db87f8..1ef4ace3e 100755 --- a/lmms_eval/tasks/worldqa/worldqa_mc.yaml +++ b/lmms_eval/tasks/worldqa/worldqa_mc.yaml @@ -10,7 +10,7 @@ metric_list: - metric: gpt_eval aggregation: !function utils.worldqa_aggregate_mc_eval higher_is_better: true -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "\nAnswer with the option's letter from the given choices directly." diff --git a/lmms_eval/tasks/worldqa/worldqa_mcppl.yaml b/lmms_eval/tasks/worldqa/worldqa_mcppl.yaml index f954bf4cb..283f18644 100755 --- a/lmms_eval/tasks/worldqa/worldqa_mcppl.yaml +++ b/lmms_eval/tasks/worldqa/worldqa_mcppl.yaml @@ -8,7 +8,7 @@ doc_to_target: !function utils.worldqa_doc_to_answer_mc_ppl doc_to_choice: !function utils.worldqa_doc_to_choice metric_list: - metric: acc -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: pre_prompt: "" post_prompt: "" diff --git a/lmms_eval/tasks/youcook2/utils.py b/lmms_eval/tasks/youcook2/utils.py index f4e9af91a..2a116f1a9 100644 --- a/lmms_eval/tasks/youcook2/utils.py +++ b/lmms_eval/tasks/youcook2/utils.py @@ -44,9 +44,9 @@ def youcook2_doc_to_visual(doc): return [os.path.join(cache_dir, doc["video_path"])] -def youcook2_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs and "prompt" in model_specific_prompt_kwargs: - return model_specific_prompt_kwargs["prompt"] +def youcook2_doc_to_text(doc, lmms_eval_specific_kwargs=None): + if lmms_eval_specific_kwargs and "prompt" in lmms_eval_specific_kwargs: + return lmms_eval_specific_kwargs["prompt"] else: return "Provide a one-sentence caption for the provided video." diff --git a/lmms_eval/tasks/youcook2/youcook2_val.yaml b/lmms_eval/tasks/youcook2/youcook2_val.yaml index f574e6a39..6668abe0a 100644 --- a/lmms_eval/tasks/youcook2/youcook2_val.yaml +++ b/lmms_eval/tasks/youcook2/youcook2_val.yaml @@ -31,7 +31,7 @@ metric_list: # aggregation : !function utils.youcook2_spice # higher_is_better : true include: _default_template_yaml -model_specific_prompt_kwargs: +lmms_eval_specific_kwargs: default: prompt: Provide a one-sentence caption for the provided video. gemini_api: From 836ec83039004a943bafa586ed04d2259f0983bb Mon Sep 17 00:00:00 2001 From: Li Bo Date: Sat, 3 Aug 2024 18:17:31 +1000 Subject: [PATCH 22/36] Dev/onevision (#153) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task * Update lmms_eval configuration to use process_results_use_image and full_docs options * split tasks * chore: Update live_bench task dataset names and versions * feat: Add multi-choice parsing and processing functions This commit adds two new functions to the `egoschema/utils.py` file: `get_multi_choice_info` and `parse_multi_choice_response`. These functions are used to parse and process multi-choice responses in the Egoschema task. The `get_multi_choice_info` function extracts information about the available choices from the input document, while the `parse_multi_choice_response` function parses the generated response and returns the predicted index. These functions are essential for accurately processing multi-choice answers in the Egoschema task. * feat: Add regex-based parsing for multi-choice predictions This commit enhances the `perceptiontest_val_process_results_mc` function in the `utils.py` file. It introduces regex-based parsing to extract the predicted choice from the raw text prediction. If a match is found for A, B, C, or D, the matched letter is used as the prediction. Otherwise, an empty string is set as the prediction. This improvement ensures accurate processing of multi-choice predictions in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Improve accuracy calculation in perception test validation This commit refactors the `perceptiontest_val_aggregate_accuracy` function in the `utils.py` file. Instead of comparing the string representations of `answer_id` and `pred_id`, it now directly checks the `correct` field in the `accuracy` dictionary. This change ensures more accurate calculation of the overall accuracy in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * Refactor accuracy calculation in perception test validation * feat: Add SRT_API model to available models This commit adds the SRT_API model to the list of available models in the `__init__.py` file. This model can now be used for evaluation and testing purposes. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * chore: Update live_bench task dataset names and versions (cherry picked from commit 46a85b40b013503e52d007c96ca0607bd4604a3e) * refactor: Update question_for_eval key in MathVerseEvaluator This commit updates the key "question_for_eval" to "question" in the MathVerseEvaluator class. This change ensures consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update generate_submission_file function in mathverse utils This commit updates the `generate_submission_file` function in the `mathverse/utils.py` file. It adds the `problem_version` to the filename to ensure that the submission files are saved with the correct problem version. This change improves the organization and clarity of the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files This commit updates the default template YAML files in the `lmms_eval/tasks` directory. It modifies the `generation_kwargs` and `metadata` sections to improve the configuration and versioning of the tasks. These changes ensure consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Add llava_onevision model to seedbench_2.yaml This commit adds the `llava_onevision` model to the `seedbench_2.yaml` file in the `lmms_eval/tasks/seedbench_2` directory. The `llava_onevision` model now has the same `img_token` and `post_prompt` properties as the other models in the file. This change ensures consistency and improves the configuration of the task. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files * refactor: Update default template YAML files * refactor: Add lazy loading for images in mvbench_episodic_reasoning.yaml This commit adds lazy loading for images in the `mvbench_episodic_reasoning.yaml` file. The `doc_to_visual` function has been updated to include lazy loading functionality, improving the performance of the page load. This change ensures faster loading times for images in the `episodic_reasoning` dataset. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata This commit updates the Task class in the lmms_eval/api/task.py file to include lazy loading for metadata. The per_task_metadata dictionary is now updated with the config.metadata only if it is not empty. This change ensures that the metadata is properly updated and avoids unnecessary updates when the config.metadata is empty. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata --------- Co-authored-by: Fanyi Pu Co-authored-by: [Co-author Name] <[coauthor@example.com]> Co-authored-by: kcz358 --- lmms_eval/api/task.py | 3 +- lmms_eval/models/llava_onevision.py | 32 ++++++++++++------- .../mvbench/mvbench_episodic_reasoning.yaml | 1 + 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index af8e81b4e..310b4bfb3 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -378,7 +378,8 @@ def build_all_requests(self, limit=None, rank=None, world_size=None) -> None: # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute per_task_metadata = {"task": self.config["task"], "doc_id": doc_id, "repeats": self.config.repeats} - if self.config.metadata: + + if self.config.metadata and type(self.config.metadata) == dict: # TODO: temporary fix for metadata loading, ignore the list of dict type. per_task_metadata.update(self.config.metadata) inst = self.construct_requests(doc_id=doc_id, ctx=fewshot_ctx, metadata=per_task_metadata, split=split) diff --git a/lmms_eval/models/llava_onevision.py b/lmms_eval/models/llava_onevision.py index 4785c5db8..3e7682257 100644 --- a/lmms_eval/models/llava_onevision.py +++ b/lmms_eval/models/llava_onevision.py @@ -454,23 +454,30 @@ def _collate(x): self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad") eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") - if "sample_frames" in metadata: + if type(visual[0]) == PIL.Image.Image and "task_type" not in metadata and "sample_frames" not in metadata: # For image task + image_tensor = process_images(visual, self._image_processor, self._config) + if type(image_tensor) is list: + image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor] + else: + image_tensor = image_tensor.to(dtype=torch.float16, device=self.device) + + task_type = "image" + placeholder_count = len(visual) if isinstance(visual, list) else 1 + + elif "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: + assert type(visual) == list, "sample_frames must be specified for video task" sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int) visual = [visual[i] for i in sample_indices] assert len(visual) == metadata["sample_frames"] - # if (len(visual) > 1 or "image_aspect_ratio" not in self._config.__dict__) and ("image_aspect_ratio" in gen_kwargs.keys()): - # self._config.image_aspect_ratio = gen_kwargs["image_aspect_ratio"] - # eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") - - if type(visual[0]) == PIL.Image.Image: # For image task image_tensor = process_images(visual, self._image_processor, self._config) if type(image_tensor) is list: image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor] else: image_tensor = image_tensor.to(dtype=torch.float16, device=self.device) - task_type = "image" + task_type = "video" + placeholder_count = 1 elif type(visual[0]) == str: # For video task image_tensor = [] @@ -486,6 +493,7 @@ def _collate(x): image_tensor = None task_type = "video" + placeholder_count = len(frames) if self.token_strategy == "multiple" else 1 if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context: """ @@ -495,11 +503,11 @@ def _collate(x): 3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line. 4. For video tasks, we could add a token or multiple tokens for each frame in the context. This depends on the training strategy and should balance in test to decide which is better """ - if task_type == "image": - image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN] - elif task_type == "video": - image_tokens = [DEFAULT_IMAGE_TOKEN] * len(frames) if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN] - + # if task_type == "image": # indeed in multi-image case, not the video in frames. + # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN] + # elif task_type == "video": + # image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count if self.token_strategy == "multiple" else [DEFAULT_IMAGE_TOKEN] + image_tokens = [DEFAULT_IMAGE_TOKEN] * placeholder_count image_tokens = " ".join(image_tokens) question = image_tokens + "\n" + context else: diff --git a/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml b/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml index 986c65bcc..9500d28ff 100644 --- a/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml +++ b/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml @@ -16,4 +16,5 @@ lmms_eval_specific_kwargs: post_prompt: "Answer with the option's letter from the given choices directly." metadata: + task_type: video sample_frames: 32 # for episodic reasoning, it stores the frame images in folder so we need to set the number of frames to sample here. \ No newline at end of file From 8745b929132a68b39e2220265a34df960e322b13 Mon Sep 17 00:00:00 2001 From: Li Bo Date: Mon, 5 Aug 2024 16:30:02 +1000 Subject: [PATCH 23/36] Dev/onevision (#154) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task * Update lmms_eval configuration to use process_results_use_image and full_docs options * split tasks * chore: Update live_bench task dataset names and versions * feat: Add multi-choice parsing and processing functions This commit adds two new functions to the `egoschema/utils.py` file: `get_multi_choice_info` and `parse_multi_choice_response`. These functions are used to parse and process multi-choice responses in the Egoschema task. The `get_multi_choice_info` function extracts information about the available choices from the input document, while the `parse_multi_choice_response` function parses the generated response and returns the predicted index. These functions are essential for accurately processing multi-choice answers in the Egoschema task. * feat: Add regex-based parsing for multi-choice predictions This commit enhances the `perceptiontest_val_process_results_mc` function in the `utils.py` file. It introduces regex-based parsing to extract the predicted choice from the raw text prediction. If a match is found for A, B, C, or D, the matched letter is used as the prediction. Otherwise, an empty string is set as the prediction. This improvement ensures accurate processing of multi-choice predictions in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Improve accuracy calculation in perception test validation This commit refactors the `perceptiontest_val_aggregate_accuracy` function in the `utils.py` file. Instead of comparing the string representations of `answer_id` and `pred_id`, it now directly checks the `correct` field in the `accuracy` dictionary. This change ensures more accurate calculation of the overall accuracy in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * Refactor accuracy calculation in perception test validation * feat: Add SRT_API model to available models This commit adds the SRT_API model to the list of available models in the `__init__.py` file. This model can now be used for evaluation and testing purposes. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * chore: Update live_bench task dataset names and versions (cherry picked from commit 46a85b40b013503e52d007c96ca0607bd4604a3e) * refactor: Update question_for_eval key in MathVerseEvaluator This commit updates the key "question_for_eval" to "question" in the MathVerseEvaluator class. This change ensures consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update generate_submission_file function in mathverse utils This commit updates the `generate_submission_file` function in the `mathverse/utils.py` file. It adds the `problem_version` to the filename to ensure that the submission files are saved with the correct problem version. This change improves the organization and clarity of the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files This commit updates the default template YAML files in the `lmms_eval/tasks` directory. It modifies the `generation_kwargs` and `metadata` sections to improve the configuration and versioning of the tasks. These changes ensure consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Add llava_onevision model to seedbench_2.yaml This commit adds the `llava_onevision` model to the `seedbench_2.yaml` file in the `lmms_eval/tasks/seedbench_2` directory. The `llava_onevision` model now has the same `img_token` and `post_prompt` properties as the other models in the file. This change ensures consistency and improves the configuration of the task. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files * refactor: Update default template YAML files * refactor: Add lazy loading for images in mvbench_episodic_reasoning.yaml This commit adds lazy loading for images in the `mvbench_episodic_reasoning.yaml` file. The `doc_to_visual` function has been updated to include lazy loading functionality, improving the performance of the page load. This change ensures faster loading times for images in the `episodic_reasoning` dataset. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata This commit updates the Task class in the lmms_eval/api/task.py file to include lazy loading for metadata. The per_task_metadata dictionary is now updated with the config.metadata only if it is not empty. This change ensures that the metadata is properly updated and avoids unnecessary updates when the config.metadata is empty. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata * refactor: Update Task class to include lazy loading for metadata * refactor: Update SRT_API class to improve image loading performance --------- Co-authored-by: Fanyi Pu Co-authored-by: [Co-author Name] <[coauthor@example.com]> Co-authored-by: kcz358 --- lmms_eval/models/srt_api.py | 47 ++++++++++++------- .../tasks/videochatgpt/_videochatgpt.yaml | 6 +-- lmms_eval/tasks/videomme/utils.py | 3 +- lmms_eval/tasks/videomme/videomme.yaml | 2 +- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py index 239ec4cf9..4650c6393 100755 --- a/lmms_eval/models/srt_api.py +++ b/lmms_eval/models/srt_api.py @@ -18,6 +18,8 @@ from loguru import logger as eval_logger +NUM_SECONDS_TO_SLEEP = 5 + @register_model("srt_api") class SRT_API(lmms): @@ -28,7 +30,7 @@ def __init__( modality: str = "video", host: str = "127.0.0.1", port: int = 30000, - max_frames_num: int = 10, + max_frames_num: int = 32, timeout: int = 120, continual_mode: bool = False, response_persistent_folder: str = None, @@ -95,6 +97,7 @@ def encode_video(self, video_path, for_get_frames_num): frames = vr.get_batch(frame_idx).asnumpy() except: import av + container = av.open(video_path) frames = [] @@ -156,7 +159,7 @@ def generate_until(self, requests) -> List[str]: eval_logger.error(f"Exception : {e} \n When loading video {visual}") imgs = None break - + # Handling video decode error # If we can't even load using pyav, then we will skip if imgs is None: @@ -166,22 +169,30 @@ def generate_until(self, requests) -> List[str]: continue messages = [] - if self.image_token not in contexts: # single image format - content = [] - for img in imgs: - content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}) - - content.append({"type": "text", "text": contexts}) - messages.append({"role": "user", "content": content}) - else: # interleaved format - contexts = contexts.split(self.image_token) - for idx, img in enumerate(imgs): - content = [ - {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}, - {"type": "text", "text": contexts[idx]}, - ] - messages.append({"role": "user", "content": content}) - messages.append({"role": "user", "content": [{"type": "text", "text": contexts[-1]}]}) + + # put the images in the first place + content = [] + for img in imgs: + content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}) + + content.append({"type": "text", "text": contexts}) + messages.append({"role": "user", "content": content}) + # if self.image_token not in contexts: # single image format + # content = [] + # for img in imgs: + # content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}) + + # content.append({"type": "text", "text": contexts}) + # messages.append({"role": "user", "content": content}) + # else: # interleaved format + # contexts = contexts.split(self.image_token) + # for idx, img in enumerate(imgs): + # content = [ + # {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}, + # {"type": "text", "text": contexts[idx]}, + # ] + # messages.append({"role": "user", "content": content}) + # messages.append({"role": "user", "content": [{"type": "text", "text": contexts[-1]}]}) if "max_new_tokens" not in gen_kwargs: gen_kwargs["max_new_tokens"] = 1024 diff --git a/lmms_eval/tasks/videochatgpt/_videochatgpt.yaml b/lmms_eval/tasks/videochatgpt/_videochatgpt.yaml index b29f95e9d..d89092320 100755 --- a/lmms_eval/tasks/videochatgpt/_videochatgpt.yaml +++ b/lmms_eval/tasks/videochatgpt/_videochatgpt.yaml @@ -1,5 +1,5 @@ group: videochatgpt task: -- videochatgpt_gen -- videochatgpt_temporal -- videochatgpt_consistency + - videochatgpt_gen + - videochatgpt_temporal + - videochatgpt_consistency diff --git a/lmms_eval/tasks/videomme/utils.py b/lmms_eval/tasks/videomme/utils.py index 98a9cbff1..d0c7e93c4 100644 --- a/lmms_eval/tasks/videomme/utils.py +++ b/lmms_eval/tasks/videomme/utils.py @@ -194,7 +194,8 @@ def videomme_doc_to_text(doc, lmms_eval_specific_kwargs=None): question = doc["question"] option = str(doc["options"]) question = question + "\n" + option - full_prompt = option_prompt + "\n" + question + "\n" + "The best answer is:" + post_prompt = lmms_eval_specific_kwargs["post_prompt"] if "post_prompt" in lmms_eval_specific_kwargs else "The best answer is:" + full_prompt = option_prompt + "\n" + question + "\n" + post_prompt return full_prompt diff --git a/lmms_eval/tasks/videomme/videomme.yaml b/lmms_eval/tasks/videomme/videomme.yaml index cc40dd0cc..08f3de195 100644 --- a/lmms_eval/tasks/videomme/videomme.yaml +++ b/lmms_eval/tasks/videomme/videomme.yaml @@ -26,7 +26,7 @@ metric_list: lmms_eval_specific_kwargs: default: pre_prompt: "" - post_prompt: "\nAnswer the question with A, B, C, or D." + post_prompt: "\nAnswer with the option's letter from the given choices directly." gpt4v: pre_prompt: "" post_prompt: "\nAnswer the question with A, B, C, or D." From 7d2a8cdbe518a1c95ef3cbc15bab99619f07d566 Mon Sep 17 00:00:00 2001 From: Li Bo Date: Wed, 7 Aug 2024 10:30:57 +1000 Subject: [PATCH 24/36] Dev/onevision (#155) * claude auto detect json mode * extract information * use claude to generate * fix bugs * fix * generate data * chore: Update dataset name and version for live_bench task * gpt-4-turbo => gpt-4o * chore: Update dataset capture settings in create_dataset.py * everything use gpt-4o * websites * livebench_july * Refactor code to simplify data assignment in example.ipynb * chore: Update dataset name for live_bench task * Update lmms_eval configuration to use process_results_use_image and full_docs options * split tasks * chore: Update live_bench task dataset names and versions * feat: Add multi-choice parsing and processing functions This commit adds two new functions to the `egoschema/utils.py` file: `get_multi_choice_info` and `parse_multi_choice_response`. These functions are used to parse and process multi-choice responses in the Egoschema task. The `get_multi_choice_info` function extracts information about the available choices from the input document, while the `parse_multi_choice_response` function parses the generated response and returns the predicted index. These functions are essential for accurately processing multi-choice answers in the Egoschema task. * feat: Add regex-based parsing for multi-choice predictions This commit enhances the `perceptiontest_val_process_results_mc` function in the `utils.py` file. It introduces regex-based parsing to extract the predicted choice from the raw text prediction. If a match is found for A, B, C, or D, the matched letter is used as the prediction. Otherwise, an empty string is set as the prediction. This improvement ensures accurate processing of multi-choice predictions in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Improve accuracy calculation in perception test validation This commit refactors the `perceptiontest_val_aggregate_accuracy` function in the `utils.py` file. Instead of comparing the string representations of `answer_id` and `pred_id`, it now directly checks the `correct` field in the `accuracy` dictionary. This change ensures more accurate calculation of the overall accuracy in the perception test validation. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * Refactor accuracy calculation in perception test validation * feat: Add SRT_API model to available models This commit adds the SRT_API model to the list of available models in the `__init__.py` file. This model can now be used for evaluation and testing purposes. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * chore: Update live_bench task dataset names and versions (cherry picked from commit 46a85b40b013503e52d007c96ca0607bd4604a3e) * refactor: Update question_for_eval key in MathVerseEvaluator This commit updates the key "question_for_eval" to "question" in the MathVerseEvaluator class. This change ensures consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update generate_submission_file function in mathverse utils This commit updates the `generate_submission_file` function in the `mathverse/utils.py` file. It adds the `problem_version` to the filename to ensure that the submission files are saved with the correct problem version. This change improves the organization and clarity of the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files This commit updates the default template YAML files in the `lmms_eval/tasks` directory. It modifies the `generation_kwargs` and `metadata` sections to improve the configuration and versioning of the tasks. These changes ensure consistency and clarity in the code. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Add llava_onevision model to seedbench_2.yaml This commit adds the `llava_onevision` model to the `seedbench_2.yaml` file in the `lmms_eval/tasks/seedbench_2` directory. The `llava_onevision` model now has the same `img_token` and `post_prompt` properties as the other models in the file. This change ensures consistency and improves the configuration of the task. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update default template YAML files * refactor: Update default template YAML files * refactor: Add lazy loading for images in mvbench_episodic_reasoning.yaml This commit adds lazy loading for images in the `mvbench_episodic_reasoning.yaml` file. The `doc_to_visual` function has been updated to include lazy loading functionality, improving the performance of the page load. This change ensures faster loading times for images in the `episodic_reasoning` dataset. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata This commit updates the Task class in the lmms_eval/api/task.py file to include lazy loading for metadata. The per_task_metadata dictionary is now updated with the config.metadata only if it is not empty. This change ensures that the metadata is properly updated and avoids unnecessary updates when the config.metadata is empty. Co-authored-by: [Co-author Name] <[coauthor@example.com]> * refactor: Update Task class to include lazy loading for metadata * refactor: Update Task class to include lazy loading for metadata * refactor: Update SRT_API class to improve image loading performance * refactor: Update SRT_API timeout to 60 seconds --------- Co-authored-by: Fanyi Pu Co-authored-by: [Co-author Name] <[coauthor@example.com]> Co-authored-by: kcz358 --- lmms_eval/models/srt_api.py | 2 +- lmms_eval/tasks/egoschema/utils.py | 2 +- lmms_eval/tasks/mathverse/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py index 4650c6393..ca9d9e900 100755 --- a/lmms_eval/models/srt_api.py +++ b/lmms_eval/models/srt_api.py @@ -31,7 +31,7 @@ def __init__( host: str = "127.0.0.1", port: int = 30000, max_frames_num: int = 32, - timeout: int = 120, + timeout: int = 60, continual_mode: bool = False, response_persistent_folder: str = None, **kwargs, diff --git a/lmms_eval/tasks/egoschema/utils.py b/lmms_eval/tasks/egoschema/utils.py index fc5ac8d5b..d771ca1b9 100755 --- a/lmms_eval/tasks/egoschema/utils.py +++ b/lmms_eval/tasks/egoschema/utils.py @@ -7,6 +7,7 @@ import json import yaml +import random from pathlib import Path with open(Path(__file__).parent / "_default_template_yaml", "r") as f: @@ -137,7 +138,6 @@ def parse_multi_choice_response(response, all_choices, index2ans): index_ans = False # it's content ans. if len(candidates) == 0: # still not get answer, randomly choose one. - # import pdb; pdb.set_trace() pred_index = random.choice(all_choices) elif len(candidates) > 1: # candidates = list(set(candidates)) diff --git a/lmms_eval/tasks/mathverse/utils.py b/lmms_eval/tasks/mathverse/utils.py index 1f1302e52..5ad391150 100644 --- a/lmms_eval/tasks/mathverse/utils.py +++ b/lmms_eval/tasks/mathverse/utils.py @@ -79,7 +79,7 @@ def mathverse_aggregate_results_submission(results, args, *, calculate_gain=Fals def mathverse_aggregate_results_eval(results, args, *, calculate_gain=False, random_scores=None): split_flag = results[0]["metadata"]["split"] - problem_version = results[0]["metadata"]["problem_version"].lower().replace(" ", "_") + problem_version = results[0]["problem_version"].lower().replace(" ", "_") # save the result first, in case the gpt evaluation fails path = generate_submission_file(f"mathverse_{split_flag}_{problem_version}_results.json", args) with open(path, "w") as f: From 6130158332214a5b036a2b37cd82125151761571 Mon Sep 17 00:00:00 2001 From: Bo Li Date: Mon, 12 Aug 2024 06:35:17 +0000 Subject: [PATCH 25/36] add MMMU Pro --- lmms_eval/models/llava_vid.py | 2 + lmms_eval/models/xcomposer2d5.py | 6 +- .../tasks/mmmu_pro/_default_template_yaml | 6 + lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml | 5 + .../tasks/mmmu_pro/mmmu_pro_composite.yaml | 17 + .../tasks/mmmu_pro/mmmu_pro_original.yaml | 17 + lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml | 17 + lmms_eval/tasks/mmmu_pro/utils.py | 492 ++++++++++++++++++ 8 files changed, 558 insertions(+), 4 deletions(-) create mode 100644 lmms_eval/tasks/mmmu_pro/_default_template_yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/utils.py diff --git a/lmms_eval/models/llava_vid.py b/lmms_eval/models/llava_vid.py index 27da7a949..c99d1576b 100755 --- a/lmms_eval/models/llava_vid.py +++ b/lmms_eval/models/llava_vid.py @@ -28,11 +28,13 @@ try: from llavavid.model.language_model.llava_qwen import LlavaQwenConfig + AutoConfig.register("llava_qwen", LlavaQwenConfig) except ImportError: eval_logger.debug("No Qwen for llava vid") from llavavid.model.language_model.llava_llama import LlavaConfig + AutoConfig.register("llava_llama", LlavaConfig) diff --git a/lmms_eval/models/xcomposer2d5.py b/lmms_eval/models/xcomposer2d5.py index 17ed8fa25..8bfe6c653 100644 --- a/lmms_eval/models/xcomposer2d5.py +++ b/lmms_eval/models/xcomposer2d5.py @@ -40,7 +40,6 @@ def __init__( if not os.path.exists(self.tmp_folder): os.makedirs(self.tmp_folder) eval_logger.info(f"Using temporary folder: {self.tmp_folder}") - batch_size = int(batch_size) assert batch_size == 1, f"Batch size should be 1 for InternVL2, but got {batch_size}." @@ -57,7 +56,7 @@ def __init__( else: self._device = torch.device(f"cuda:{accelerator.local_process_index}") self.device_map = f"cuda:{accelerator.local_process_index}" - + self.path = pretrained self._model = AutoModel.from_pretrained(self.path, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map=self.device_map).half().eval() self._tokenizer = AutoTokenizer.from_pretrained(self.path, trust_remote_code=True) @@ -130,7 +129,6 @@ def rank(self): def world_size(self): return self._world_size - def flatten(self, input): new_list = [] for i in input: @@ -168,7 +166,7 @@ def generate_until(self, requests) -> List[str]: gen_kwargs["num_beams"] = 1 try: - with torch.autocast(device_type='cuda', dtype=torch.float16): + with torch.autocast(device_type="cuda", dtype=torch.float16): response, his = self.model.chat(self.tokenizer, contexts, image, do_sample=False, num_beams=1, use_meta=True, max_new_tokens=gen_kwargs["max_new_tokens"]) except Exception as e: eval_logger.error(f"Error : {e}") diff --git a/lmms_eval/tasks/mmmu_pro/_default_template_yaml b/lmms_eval/tasks/mmmu_pro/_default_template_yaml new file mode 100644 index 000000000..6dbeb3c35 --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/_default_template_yaml @@ -0,0 +1,6 @@ +generation_kwargs: + max_new_tokens: 256 + +metadata: + version: 0.0 + interleaved_format: false \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml new file mode 100755 index 000000000..a374f5c39 --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml @@ -0,0 +1,5 @@ +group: mmmu_pro +task: +- mmmu_pro_vision +- mmmu_pro_composite +- mmmu_pro_original diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml new file mode 100755 index 000000000..6c46c0135 --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml @@ -0,0 +1,17 @@ +task: "mmmu_pro_composite" +dataset_path: MMMU/MMMU_Pro +dataset_name: composite +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mmmu_pro_doc_to_visual +doc_to_text: "Answer the series of questions individually with the option letters from the given choices in the image. The questions are ordered from top to bottom and left to right." +doc_to_target: "{{answers}}" +# The return value of process_results will be used by metrics +process_results: !function utils.mmmu_pro_composite_process_results + +metric_list: + - metric: mmmu_acc + aggregation: !function utils.mmmu_pro_aggregate_results + higher_is_better: true + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml new file mode 100755 index 000000000..872332d9d --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml @@ -0,0 +1,17 @@ +task: "mmmu_pro_original" +dataset_path: MMMU/MMMU_Pro +dataset_name: original +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mmmu_pro_doc_to_visual +doc_to_text: !function utils.mmmu_pro_doc_to_text +doc_to_target: "{{answer}}" +# The return value of process_results will be used by metrics +process_results: !function utils.mmmu_pro_process_results + +metric_list: + - metric: mmmu_acc + aggregation: !function utils.mmmu_pro_aggregate_results + higher_is_better: true + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml new file mode 100755 index 000000000..0073b5eef --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml @@ -0,0 +1,17 @@ +task: "mmmu_pro_vision" +dataset_path: MMMU/MMMU_Pro +dataset_name: vision +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mmmu_pro_doc_to_visual +doc_to_text: !function utils.mmmu_pro_doc_to_text +doc_to_target: "{{answer}}" +# The return value of process_results will be used by metrics +process_results: !function utils.mmmu_pro_process_results + +metric_list: + - metric: mmmu_acc + aggregation: !function utils.mmmu_pro_aggregate_results + higher_is_better: true + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/utils.py b/lmms_eval/tasks/mmmu_pro/utils.py new file mode 100755 index 000000000..082e832cb --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/utils.py @@ -0,0 +1,492 @@ +from collections import defaultdict +import re +import ast +import random +import numpy as np +import os +import json +from pathlib import Path +import yaml + +from lmms_eval.tasks._task_utils.file_utils import generate_submission_file + +from loguru import logger as eval_logger + +MULTI_CHOICE_PROMPT = "Answer with the option's letter from the given choices directly." +OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase." + +with open(Path(__file__).parent / "_default_template_yaml", "r") as f: + raw_data = f.readlines() + safe_data = [] + for i, line in enumerate(raw_data): + # remove function definition since yaml load cannot handle it + if "!function" not in line: + safe_data.append(line) + + config = yaml.safe_load("".join(safe_data)) + + +def replace_images_tokens(input_string): + for i in range(1, 8): + question_text = f"" + query_text = "" + if question_text in input_string: + input_string = input_string.replace(question_text, query_text) + return input_string + + +def parse_options(options): + option_letters = [chr(ord("A") + i) for i in range(len(options))] + choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)]) + return choices_str + + +def construct_prompt(doc): + question = doc["question"] + # Weirdly, data["options"] is a string in MMMU Huggingface dataset + parsed_options = parse_options(ast.literal_eval(doc["options"])) + # parsed_options already prepends a newline so no need to add space here + question = f"{question}\n{parsed_options}\n\n{MULTI_CHOICE_PROMPT}" + return question + + +def mmmu_pro_doc_to_text(doc): + if "question" in doc and "options" in doc: # original operation + question = construct_prompt(doc) + if config["metadata"]["interleaved_format"]: + question = replace_images_tokens(question) + else: # vision-only operation + question = "Please answer the question in this image with the option's letter from the given choices directly." + return question + + +def mmmu_pro_doc_to_visual(doc): + if "question" in doc and "options" in doc: # original operation + prompt = construct_prompt(doc) + image_tokens = re.findall(r"", prompt) + # Remove <> and swap space as _ + image_tokens = sorted(list(set([image_token.strip("<>").replace(" ", "_") for image_token in image_tokens]))) + visual = [doc[image_token].convert("RGB") for image_token in image_tokens] + else: # vision-only operation + visual = [doc["image"].convert("RGB")] + return visual + + +# MMMU-PRO's all questions are multiple-choice questions +def mmmu_pro_process_results(doc, results): + pred = results[0] + if "question" in doc and "options" in doc: + index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"])) + parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans) + else: + parsed_pred = pred + + mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": doc["answer"], "parsed_pred": parsed_pred} + return {"mmmu_acc": mmmu_acc} + + +def mmmu_pro_composite_process_results(doc, results): + pred = results[0] + gt_list = ast.literal_eval(doc["answers"]) + # Parse out option letters from the prediction + option_letters = re.findall(r"\b[A-Z]\b", pred) + + cutout_letters = option_letters[: len(gt_list)] + + # If cutout_letters is shorter than gt_list, pad with empty strings + while len(cutout_letters) < len(gt_list): + cutout_letters.append("") + + assert len(cutout_letters) == len(gt_list), f"Mismatch in lengths: cutout_letters ({len(cutout_letters)}) != gt_list ({len(gt_list)})" + + mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": gt_list, "parsed_pred": cutout_letters} + return {"mmmu_acc": mmmu_acc} + + +def mmmu_pro_aggregate_results(results): + evaluation_result = {} + subset_to_eval_samples = defaultdict(list) + for result in results: + subset_to_eval_samples[result["subject"]].append(result) + + for subset, sub_eval_samples in subset_to_eval_samples.items(): + judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples) + metric_dict.update({"num_example": len(sub_eval_samples)}) + evaluation_result[subset] = metric_dict + + printable_results = {} + for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): + in_domain_cat_results = {} + for cat_name in in_domain_cats: + if cat_name in evaluation_result.keys(): + in_domain_cat_results[cat_name] = evaluation_result[cat_name] + else: + pass + in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) + in_domain_data_num = sum([cat_results["num_example"] for cat_results in in_domain_cat_results.values()]) + printable_results["Overall-" + domain] = { + "num": int(in_domain_data_num), + "acc": round(in_domain_ins_acc, 5), + } + # add sub category + for cat_name, cat_results in in_domain_cat_results.items(): + printable_results[cat_name] = { + "num": int(cat_results["num_example"]), + "acc": round(cat_results["acc"], 5), + } + + all_ins_acc = calculate_ins_level_acc(evaluation_result) + printable_results["Overall"] = { + "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]), + "acc": round(all_ins_acc, 5), + } + print(printable_results) + return printable_results["Overall"]["acc"] + + +################## +# Helper functions written by official MMMU repo. +################## + + +def calculate_ins_level_acc(results): + """Calculate the instruction level accuracy for given Subject results + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246 + """ + acc = 0 + ins_num = 0 + for cat_results in results.values(): + acc += cat_results["acc"] * cat_results["num_example"] + ins_num += cat_results["num_example"] + if ins_num == 0: + return 0 + return acc / ins_num + + +DOMAIN_CAT2SUB_CAT = { + "Art and Design": ["Art", "Art_Theory", "Design", "Music"], + "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], + "Science": [ + "Biology", + "Chemistry", + "Geography", + "Math", + "Physics", + ], + "Health and Medicine": [ + "Basic_Medical_Science", + "Clinical_Medicine", + "Diagnostics_and_Laboratory_Medicine", + "Pharmacy", + "Public_Health", + ], + "Humanities and Social Science": [ + "History", + "Literature", + "Sociology", + "Psychology", + ], + "Tech and Engineering": [ + "Agriculture", + "Architecture_and_Engineering", + "Computer_Science", + "Electronics", + "Energy_and_Power", + "Materials", + "Mechanical_Engineering", + ], +} + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175 + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191 + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct + + +def evaluate_mmmu(samples): + """ + Batch evaluation for multiple choice and open questions. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219 + """ + total_count = 0 + pred_correct = 0 + judge_dict = dict() + for sample in samples: + if isinstance(sample["answer"], str) and isinstance(sample["parsed_pred"], str): + gold_i = sample["answer"] + pred_i = sample["parsed_pred"] + correct = eval_multi_choice(gold_i, pred_i) + total_count += 1 + if correct: + judge_dict[sample["id"]] = "Correct" + pred_correct += 1 + else: + judge_dict[sample["id"]] = "Wrong" + elif isinstance(sample["answer"], list) and isinstance(sample["parsed_pred"], list): + for gold, pred in zip(sample["answer"], sample["parsed_pred"]): + correct = eval_multi_choice(gold, pred) + total_count += 1 + if correct: + judge_dict[sample["id"]] = "Correct" + pred_correct += 1 + else: + judge_dict[sample["id"]] = "Wrong" + + if total_count == 0: + return {"acc": 0} + return judge_dict, {"acc": pred_correct / total_count} + + +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10 + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f"{choice} " in response: + candidates.append(choice) + + if len(candidates) == 0: + for choice in all_choices: # e.g., A. B. C. D. + if f"{choice}." in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + return pred_index + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100 + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbersz + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def check_is_number(string): + """ + Check if the given string a number. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65 + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76 + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122 + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +def get_multi_choice_info(options): + """ + Given the list of options for multiple choice question + Return the index2ans and all_choices + https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54 + """ + + start_chr = "A" + all_choices = [] + index2ans = {} + for i, option in enumerate(options): + index2ans[chr(ord(start_chr) + i)] = option + all_choices.append(chr(ord(start_chr) + i)) + + return index2ans, all_choices From 4ea709e3cec1c9e751190b0e43b82ee569f3a63a Mon Sep 17 00:00:00 2001 From: Bo Li Date: Mon, 19 Aug 2024 02:46:16 +0000 Subject: [PATCH 26/36] feat: Add MMMU Pro task and related YAML files --- lmms_eval/tasks/egoschema/utils.py | 2 +- .../longvideobench/longvideobench_test_i.yaml | 29 +++++++++++++++++++ .../longvideobench/longvideobench_test_v.yaml | 28 ++++++++++++++++++ lmms_eval/tasks/longvideobench/utils.py | 11 ++++++- .../tasks/mmmu_pro/mmmu_pro_composite.yaml | 2 +- .../mmmu_pro/mmmu_pro_composite_cot.yaml | 17 +++++++++++ lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml | 5 ++++ .../tasks/mmmu_pro/mmmu_pro_original.yaml | 7 ++++- .../tasks/mmmu_pro/mmmu_pro_original_cot.yaml | 22 ++++++++++++++ lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml | 2 +- .../tasks/mmmu_pro/mmmu_pro_vision_cot.yaml | 17 +++++++++++ lmms_eval/tasks/mmmu_pro/utils.py | 14 ++++----- 12 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 lmms_eval/tasks/longvideobench/longvideobench_test_i.yaml create mode 100644 lmms_eval/tasks/longvideobench/longvideobench_test_v.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_composite_cot.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_original_cot.yaml create mode 100755 lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml diff --git a/lmms_eval/tasks/egoschema/utils.py b/lmms_eval/tasks/egoschema/utils.py index d771ca1b9..86bcc8d38 100755 --- a/lmms_eval/tasks/egoschema/utils.py +++ b/lmms_eval/tasks/egoschema/utils.py @@ -23,7 +23,7 @@ # We will unzip all the zip files # To HF HOME cache dir # And load it here -HF_HOME = os.environ["HF_HOME"] +HF_HOME = os.environ["HF_HOME"] if "HF_HOME" in os.environ else os.path.expanduser("~/.cache/huggingface/hub") cache_dir = config["dataset_kwargs"]["cache_dir"] cache_dir = os.path.join(HF_HOME, cache_dir) cache_dir = os.path.join(cache_dir, "videos") diff --git a/lmms_eval/tasks/longvideobench/longvideobench_test_i.yaml b/lmms_eval/tasks/longvideobench/longvideobench_test_i.yaml new file mode 100644 index 000000000..42475adfe --- /dev/null +++ b/lmms_eval/tasks/longvideobench/longvideobench_test_i.yaml @@ -0,0 +1,29 @@ +dataset_path: longvideobench/LongVideoBench +dataset_kwargs: + token: True + cache_dir: longvideobench + video: True + force_download: False + local_files_only: False + # From_YouTube: True +task: longvideobench_test_i +test_split: test +doc_to_visual: !function utils.longvideobench_doc_to_visual_i +doc_to_text: !function utils.longvideobench_doc_to_text +doc_to_target: "correct_choice" +generation_kwargs: + max_new_tokens: 32 + temperature: 0 + do_sample: False +process_results: !function utils.longvideobench_process_results +metric_list: + - metric: submission + aggregation: !function utils.longvideobench_aggregate_results_for_submission + higher_is_better: true + +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "Answer with the option's letter from the given choices directly.\n" + insert_interleave_subtitles: True + \ No newline at end of file diff --git a/lmms_eval/tasks/longvideobench/longvideobench_test_v.yaml b/lmms_eval/tasks/longvideobench/longvideobench_test_v.yaml new file mode 100644 index 000000000..40bddf763 --- /dev/null +++ b/lmms_eval/tasks/longvideobench/longvideobench_test_v.yaml @@ -0,0 +1,28 @@ +dataset_path: longvideobench/LongVideoBench +dataset_kwargs: + token: True + cache_dir: longvideobench + video: True + force_download: False + local_files_only: False + # From_YouTube: True +task: longvideobench_test_v +test_split: test +doc_to_visual: !function utils.longvideobench_doc_to_visual_v +doc_to_text: !function utils.longvideobench_doc_to_text +doc_to_target: "correct_choice" +generation_kwargs: + max_new_tokens: 32 + temperature: 0 + do_sample: False +process_results: !function utils.longvideobench_process_results +metric_list: + - metric: submission + aggregation: !function utils.longvideobench_aggregate_results_for_submission + higher_is_better: true + +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "Answer with the option's letter from the given choices directly.\n" + \ No newline at end of file diff --git a/lmms_eval/tasks/longvideobench/utils.py b/lmms_eval/tasks/longvideobench/utils.py index 5d1bad20c..d10cbe67d 100644 --- a/lmms_eval/tasks/longvideobench/utils.py +++ b/lmms_eval/tasks/longvideobench/utils.py @@ -19,6 +19,7 @@ import re import json +from loguru import logger as eval_logger def timestamp_to_seconds(timestamp): @@ -355,5 +356,13 @@ def longvideobench_aggregate_results(results): "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]), "acc": round(all_ins_acc, 5), } - print(printable_results) + eval_logger.info(printable_results) return printable_results["Overall"]["acc"] + + +def longvideobench_aggregate_results_for_submission(results, args): + path = generate_submission_file("longvideobench_test_for_submission.json", args) + results_dict = {list(item.keys())[0]: list(item.values())[0] for item in results} + with open(path, "w") as f: + json.dump(results_dict, f) + eval_logger.info(f"Results saved to {path}.") diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml index 6c46c0135..59468f5d9 100755 --- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite.yaml @@ -4,7 +4,7 @@ dataset_name: composite test_split: test output_type: generate_until doc_to_visual: !function utils.mmmu_pro_doc_to_visual -doc_to_text: "Answer the series of questions individually with the option letters from the given choices in the image. The questions are ordered from top to bottom and left to right." +doc_to_text: "Answer with the option letter for each multiple choice question from the given choices directly. For each question, the last line of your response should be in the following format: 'No: [id]\nAnswer: [LETTER]' (without quotes) where LETTER is one of options." doc_to_target: "{{answers}}" # The return value of process_results will be used by metrics process_results: !function utils.mmmu_pro_composite_process_results diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite_cot.yaml new file mode 100755 index 000000000..350cb5203 --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_composite_cot.yaml @@ -0,0 +1,17 @@ +task: "mmmu_pro_composite_cot" +dataset_path: MMMU/MMMU_Pro +dataset_name: composite +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mmmu_pro_doc_to_visual +doc_to_text: "Write out all the multiple-choice questions in the image and then solve it. For each question, the last line of your response should be in the following format: 'No: [id]\nAnswer: [LETTER]' (without quotes) where LETTER is one of options. Think step by step before answering each question." +doc_to_target: "{{answers}}" +# The return value of process_results will be used by metrics +process_results: !function utils.mmmu_pro_composite_process_results + +metric_list: + - metric: mmmu_acc + aggregation: !function utils.mmmu_pro_aggregate_results + higher_is_better: true + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml new file mode 100755 index 000000000..7ca0040c6 --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml @@ -0,0 +1,5 @@ +group: mmmu_pro_cot +task: +- mmmu_pro_vision_cot +- mmmu_pro_composite_cot +- mmmu_pro_original_cot diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml index 872332d9d..0a5b8a5c4 100755 --- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_original.yaml @@ -14,4 +14,9 @@ metric_list: aggregation: !function utils.mmmu_pro_aggregate_results higher_is_better: true -include: _default_template_yaml \ No newline at end of file +include: _default_template_yaml + +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "Answer with the option letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_original_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_original_cot.yaml new file mode 100755 index 000000000..bad9f7d2b --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_original_cot.yaml @@ -0,0 +1,22 @@ +task: "mmmu_pro_original_cot" +dataset_path: MMMU/MMMU_Pro +dataset_name: original +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mmmu_pro_doc_to_visual +doc_to_text: !function utils.mmmu_pro_doc_to_text +doc_to_target: "{{answer}}" +# The return value of process_results will be used by metrics +process_results: !function utils.mmmu_pro_process_results + +metric_list: + - metric: mmmu_acc + aggregation: !function utils.mmmu_pro_aggregate_results + higher_is_better: true + +include: _default_template_yaml + +lmms_eval_specific_kwargs: + default: + pre_prompt: "" + post_prompt: "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering." \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml index 0073b5eef..4ced2cd13 100755 --- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml @@ -4,7 +4,7 @@ dataset_name: vision test_split: test output_type: generate_until doc_to_visual: !function utils.mmmu_pro_doc_to_visual -doc_to_text: !function utils.mmmu_pro_doc_to_text +doc_to_text: "Answer with the option letter from the given choices directly. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options." doc_to_target: "{{answer}}" # The return value of process_results will be used by metrics process_results: !function utils.mmmu_pro_process_results diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml new file mode 100755 index 000000000..675e9234b --- /dev/null +++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml @@ -0,0 +1,17 @@ +task: "mmmu_pro_vision_cot" +dataset_path: MMMU/MMMU_Pro +dataset_name: vision +test_split: test +output_type: generate_until +doc_to_visual: !function utils.mmmu_pro_doc_to_visual +doc_to_text: "Write out the multiple-choice question in the image and then solve it. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering." +doc_to_target: "{{answer}}" +# The return value of process_results will be used by metrics +process_results: !function utils.mmmu_pro_process_results + +metric_list: + - metric: mmmu_acc + aggregation: !function utils.mmmu_pro_aggregate_results + higher_is_better: true + +include: _default_template_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu_pro/utils.py b/lmms_eval/tasks/mmmu_pro/utils.py index 082e832cb..732904da8 100755 --- a/lmms_eval/tasks/mmmu_pro/utils.py +++ b/lmms_eval/tasks/mmmu_pro/utils.py @@ -12,9 +12,6 @@ from loguru import logger as eval_logger -MULTI_CHOICE_PROMPT = "Answer with the option's letter from the given choices directly." -OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase." - with open(Path(__file__).parent / "_default_template_yaml", "r") as f: raw_data = f.readlines() safe_data = [] @@ -41,22 +38,21 @@ def parse_options(options): return choices_str -def construct_prompt(doc): +def construct_prompt(doc, post_prompt="Answer with the option letter from the given choices directly."): question = doc["question"] # Weirdly, data["options"] is a string in MMMU Huggingface dataset parsed_options = parse_options(ast.literal_eval(doc["options"])) # parsed_options already prepends a newline so no need to add space here - question = f"{question}\n{parsed_options}\n\n{MULTI_CHOICE_PROMPT}" + question = f"{question}\n{parsed_options}\n\n{post_prompt}" return question -def mmmu_pro_doc_to_text(doc): +def mmmu_pro_doc_to_text(doc, lmms_eval_specific_kwargs=None): + post_prompt = lmms_eval_specific_kwargs["post_prompt"] if "question" in doc and "options" in doc: # original operation - question = construct_prompt(doc) + question = construct_prompt(doc, post_prompt) if config["metadata"]["interleaved_format"]: question = replace_images_tokens(question) - else: # vision-only operation - question = "Please answer the question in this image with the option's letter from the given choices directly." return question From 8b05ef644072df0d1646da96024e98c59fa1e333 Mon Sep 17 00:00:00 2001 From: Bo Li Date: Mon, 19 Aug 2024 03:39:26 +0000 Subject: [PATCH 27/36] feat: Update pre-commit hooks to include isort --- .gitignore | 1 + .pre-commit-config.yaml | 5 + lmms_eval/__main__.py | 25 ++-- lmms_eval/api/filter.py | 3 +- lmms_eval/api/metrics.py | 8 +- lmms_eval/api/model.py | 13 +- lmms_eval/api/registry.py | 6 +- lmms_eval/api/task.py | 33 ++--- lmms_eval/evaluator.py | 34 ++---- lmms_eval/filters/__init__.py | 6 +- lmms_eval/filters/extraction.py | 1 + lmms_eval/logging_utils.py | 17 +-- lmms_eval/models/__init__.py | 4 +- lmms_eval/models/batch_gpt4.py | 15 ++- lmms_eval/models/claude.py | 17 ++- lmms_eval/models/from_log.py | 12 +- lmms_eval/models/fuyu.py | 21 ++-- lmms_eval/models/gemini_api.py | 17 ++- lmms_eval/models/gpt4v.py | 18 +-- lmms_eval/models/idefics2.py | 14 +-- lmms_eval/models/instructblip.py | 21 ++-- lmms_eval/models/internvl.py | 40 +++--- lmms_eval/models/internvl2.py | 17 +-- lmms_eval/models/llama_vid.py | 35 +++--- lmms_eval/models/llava.py | 21 ++-- lmms_eval/models/llava_hf.py | 15 +-- lmms_eval/models/llava_onevision.py | 115 +++++++----------- lmms_eval/models/llava_sglang.py | 13 +- lmms_eval/models/llava_vid.py | 34 +++--- lmms_eval/models/longva.py | 30 +++-- lmms_eval/models/minicpm_v.py | 15 ++- lmms_eval/models/model_utils/load_video.py | 2 +- .../model_utils/qwen/qwen_generate_utils.py | 5 +- lmms_eval/models/mplug_owl_video.py | 20 +-- lmms_eval/models/mplug_owl_video/__init__.py | 13 +- .../configuration_mplug_owl.py | 6 +- .../mplug_owl_video/modeling_mplug_owl.py | 24 ++-- .../mplug_owl_video/processing_mplug_owl.py | 13 +- .../mplug_owl_video/tokenization_mplug_owl.py | 3 +- lmms_eval/models/phi3v.py | 13 +- lmms_eval/models/qwen_vl.py | 14 +-- lmms_eval/models/qwen_vl_api.py | 17 ++- lmms_eval/models/reka.py | 24 ++-- lmms_eval/models/srt_api.py | 34 ++---- lmms_eval/models/tinyllava.py | 16 +-- lmms_eval/models/video_chatgpt.py | 22 ++-- .../models/video_chatgpt/eval/model_utils.py | 10 +- lmms_eval/models/video_chatgpt/inference.py | 6 +- .../models/video_chatgpt/model/__init__.py | 3 +- .../models/video_chatgpt/model/consolidate.py | 3 +- .../models/video_chatgpt/model/make_delta.py | 2 +- lmms_eval/models/video_chatgpt/model/utils.py | 3 +- .../video_chatgpt/model/video_chatgpt.py | 7 +- .../video_chatgpt/single_video_inference.py | 19 +-- .../video_chatgpt/video_conversation.py | 3 +- lmms_eval/models/video_llava.py | 18 +-- lmms_eval/models/vila.py | 30 ++--- lmms_eval/models/xcomposer2_4KHD.py | 22 ++-- lmms_eval/models/xcomposer2d5.py | 19 +-- lmms_eval/tasks/__init__.py | 23 ++-- lmms_eval/tasks/activitynetqa/utils.py | 21 ++-- lmms_eval/tasks/ai2d/utils.py | 3 +- lmms_eval/tasks/chartqa/upload_chartqa.py | 4 +- lmms_eval/tasks/cmmmu/utils.py | 13 +- lmms_eval/tasks/coco_cap/utils.py | 9 +- lmms_eval/tasks/conbench/utils.py | 5 +- lmms_eval/tasks/cvrr/utils.py | 19 ++- lmms_eval/tasks/detailcaps/utils.py | 12 +- lmms_eval/tasks/docvqa/utils.py | 3 +- lmms_eval/tasks/egoschema/utils.py | 15 +-- lmms_eval/tasks/ferret/utils.py | 13 +- lmms_eval/tasks/flickr30k/utils.py | 12 +- .../tasks/hallusion_bench/evaluate_hb.py | 8 +- lmms_eval/tasks/hallusion_bench/utils.py | 8 +- lmms_eval/tasks/ii_bench/utils.py | 3 +- lmms_eval/tasks/infovqa/utils.py | 3 +- .../tasks/internal_eval/d170_cn_utils.py | 11 +- .../tasks/internal_eval/d170_en_utils.py | 11 +- .../tasks/internal_eval/dc100_en_utils.py | 12 +- .../tasks/internal_eval/dc200_cn_utils.py | 12 +- lmms_eval/tasks/live_bench/utils.py | 20 +-- lmms_eval/tasks/llava-bench-coco/utils.py | 13 +- lmms_eval/tasks/llava-in-the-wild/utils.py | 13 +- .../tasks/llava_interleave_bench/utils.py | 10 +- lmms_eval/tasks/llava_wilder/utils.py | 9 +- lmms_eval/tasks/longvideobench/utils.py | 22 ++-- lmms_eval/tasks/mathverse/mathverse_evals.py | 5 +- lmms_eval/tasks/mathverse/utils.py | 9 +- lmms_eval/tasks/mathvista/mathvista_evals.py | 5 +- lmms_eval/tasks/mathvista/utils.py | 9 +- lmms_eval/tasks/mlvu/utils.py | 16 +-- lmms_eval/tasks/mmbench/cc_utils.py | 9 +- lmms_eval/tasks/mmbench/cn_utils.py | 10 +- lmms_eval/tasks/mmbench/en_utils.py | 9 +- lmms_eval/tasks/mmbench/mmbench_evals.py | 10 +- lmms_eval/tasks/mmbench/ru_utils.py | 9 +- lmms_eval/tasks/mme/utils.py | 8 +- lmms_eval/tasks/mmmu/utils.py | 14 +-- lmms_eval/tasks/mmmu/utils_group_img.py | 14 +-- lmms_eval/tasks/mmmu_pro/utils.py | 14 +-- lmms_eval/tasks/mmstar/utils.py | 8 +- lmms_eval/tasks/mmupd/mmupd_evals.py | 13 +- lmms_eval/tasks/mmupd/utils.py | 15 +-- lmms_eval/tasks/mmvet/utils.py | 5 +- lmms_eval/tasks/muirbench/utils.py | 6 +- lmms_eval/tasks/multidocvqa/utils.py | 8 +- .../utils.py | 12 +- lmms_eval/tasks/mvbench/utils.py | 19 +-- lmms_eval/tasks/nextqa/utils.py | 11 +- lmms_eval/tasks/nocaps/utils.py | 9 +- lmms_eval/tasks/ocrbench/upload_ocrbench.py | 9 +- lmms_eval/tasks/ocrbench/utils.py | 4 +- lmms_eval/tasks/ok_vqa/_generate_config.py | 1 + lmms_eval/tasks/ok_vqa/utils.py | 13 +- lmms_eval/tasks/olympiadbench/cn_utils.py | 11 +- lmms_eval/tasks/olympiadbench/en_utils.py | 11 +- .../olympiadbench/olympiadbench_evals.py | 6 +- lmms_eval/tasks/perceptiontest/test/utils.py | 13 +- lmms_eval/tasks/perceptiontest/val/utils.py | 13 +- lmms_eval/tasks/qbench/utils.py | 1 + lmms_eval/tasks/realworldqa/utils.py | 3 +- lmms_eval/tasks/refcoco+/_generate_config.py | 1 + lmms_eval/tasks/refcoco+/utils.py | 2 +- lmms_eval/tasks/refcoco/_generate_config.py | 1 + lmms_eval/tasks/refcoco/utils.py | 2 +- lmms_eval/tasks/refcocog/_generate_config.py | 1 + lmms_eval/tasks/refcocog/utils.py | 2 +- lmms_eval/tasks/screenspot/utils.py | 2 +- lmms_eval/tasks/screenspot/utils_rec.py | 1 + lmms_eval/tasks/stvqa/utils.py | 1 + lmms_eval/tasks/synthdog/donut_evaluator.py | 4 +- lmms_eval/tasks/synthdog/utils.py | 5 +- lmms_eval/tasks/tempcompass/utils.py | 23 ++-- lmms_eval/tasks/textcaps/utils.py | 10 +- lmms_eval/tasks/textvqa/utils.py | 15 ++- lmms_eval/tasks/vatex/utils.py | 14 +-- lmms_eval/tasks/vcr_wiki/utils.py | 11 +- lmms_eval/tasks/vibe_eval/utils.py | 10 +- .../tasks/video_detail_description/utils.py | 13 +- lmms_eval/tasks/videochatgpt/utils.py | 21 ++-- lmms_eval/tasks/videomme/utils.py | 19 ++- lmms_eval/tasks/vitatecs/utils.py | 22 ++-- .../tasks/vizwiz_vqa/_generate_config.py | 1 + lmms_eval/tasks/vizwiz_vqa/utils.py | 13 +- lmms_eval/tasks/vqav2/utils.py | 13 +- lmms_eval/tasks/websrc/utils.py | 11 +- lmms_eval/tasks/wild_vision_bench/utils.py | 12 +- lmms_eval/tasks/worldqa/utils.py | 17 +-- .../tasks/worldqa/worldqa_mc_evaluator.py | 10 +- lmms_eval/tasks/youcook2/utils.py | 11 +- lmms_eval/utils.py | 45 +++---- miscs/test_llava.py | 3 +- tools/get_video_avg_time.py | 6 +- tools/live_bench/create_dataset.py | 3 +- tools/live_bench/live_bench/__init__.py | 3 +- tools/live_bench/live_bench/api/live_bench.py | 2 +- .../live_bench/data_generator/__init__.py | 5 +- .../live_bench/data_generator/live_bench.py | 30 ++--- .../data_generator/live_bench_data.py | 6 +- .../live_bench/data_generator/qa_generator.py | 30 +++-- .../data_generator/question_finalizer.py | 18 +-- .../live_bench/data_generator/score_getter.py | 17 +-- .../live_bench/data_generator/utils/claude.py | 11 +- .../utils/extract_infomation.py | 13 +- .../live_bench/data_generator/utils/gemini.py | 7 +- .../live_bench/data_generator/utils/gpt4v.py | 7 +- .../live_bench/driver/load_driver.py | 6 +- .../live_bench/screen_shoter/__init__.py | 3 +- .../live_bench/screen_shoter/screen.py | 5 +- .../live_bench/screen_shoter/screen_shoter.py | 16 +-- .../live_bench/websites/__init__.py | 3 +- .../live_bench/websites/load_website.py | 6 +- .../live_bench/live_bench/websites/website.py | 4 +- tools/live_bench/refine_all_results.py | 3 +- tools/live_bench/script/upload_results.py | 9 +- tools/make_vatex.py | 3 +- 176 files changed, 1075 insertions(+), 1093 deletions(-) diff --git a/.gitignore b/.gitignore index edf2efef2..6e98f9d8c 100755 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ ckpt pretrained/ LLaVA/ *logs +*.isorted temp/ InternVL/ logs/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a33bcacfa..97029edfe 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,4 +3,9 @@ repos: rev: 23.12.1 hooks: - id: black + language_version: python3 + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort language_version: python3 \ No newline at end of file diff --git a/lmms_eval/__main__.py b/lmms_eval/__main__.py index ef0e2f1c7..289289645 100755 --- a/lmms_eval/__main__.py +++ b/lmms_eval/__main__.py @@ -1,30 +1,29 @@ +import argparse +import datetime import importlib +import json import os -import yaml import sys -import json - import traceback -import argparse -import numpy as np -import datetime - import warnings -import traceback + +import numpy as np +import yaml warnings.simplefilter("ignore", category=DeprecationWarning) -from accelerate import Accelerator -from accelerate.utils import InitProcessGroupKwargs +import hashlib from pathlib import Path from typing import Union -import hashlib + +from accelerate import Accelerator +from accelerate.utils import InitProcessGroupKwargs +from loguru import logger as eval_logger from lmms_eval import evaluator, utils -from lmms_eval.tasks import initialize_tasks, include_path, get_task_dict from lmms_eval.api.registry import ALL_TASKS from lmms_eval.logging_utils import WandbLogger -from loguru import logger as eval_logger +from lmms_eval.tasks import get_task_dict, include_path, initialize_tasks def _handle_non_serializable(o): diff --git a/lmms_eval/api/filter.py b/lmms_eval/api/filter.py index 3fc6435dd..654ea6332 100755 --- a/lmms_eval/api/filter.py +++ b/lmms_eval/api/filter.py @@ -1,9 +1,10 @@ from dataclasses import dataclass from typing import List -from lmms_eval.api.instance import Instance from datasets import Dataset +from lmms_eval.api.instance import Instance + class Filter: """ diff --git a/lmms_eval/api/metrics.py b/lmms_eval/api/metrics.py index 157ed2df5..b3823e41d 100755 --- a/lmms_eval/api/metrics.py +++ b/lmms_eval/api/metrics.py @@ -1,16 +1,16 @@ import math +import random from collections.abc import Iterable +import evaluate import numpy as np import sacrebleu import sklearn.metrics -import random -import evaluate import torch - -from lmms_eval.api.registry import register_metric, register_aggregation from loguru import logger as eval_logger +from lmms_eval.api.registry import register_aggregation, register_metric + # Register Aggregations First @register_aggregation("bypass") diff --git a/lmms_eval/api/model.py b/lmms_eval/api/model.py index e6eeaba64..ab9dbd523 100755 --- a/lmms_eval/api/model.py +++ b/lmms_eval/api/model.py @@ -1,16 +1,15 @@ import abc +import hashlib +import json import os +from typing import List, Optional, Tuple, Type, TypeVar, Union -from typing import Union, List, Tuple, Optional, Type, TypeVar +from loguru import logger as eval_logger from sqlitedict import SqliteDict -import json -import hashlib -from lmms_eval.api.instance import Instance from tqdm import tqdm -from lmms_eval import utils - -from loguru import logger as eval_logger +from lmms_eval import utils +from lmms_eval.api.instance import Instance T = TypeVar("T", bound="lmms") diff --git a/lmms_eval/api/registry.py b/lmms_eval/api/registry.py index 556a95282..630fd4274 100755 --- a/lmms_eval/api/registry.py +++ b/lmms_eval/api/registry.py @@ -1,10 +1,10 @@ -from lmms_eval.api.model import lmms - from typing import Callable, Dict -import evaluate as hf_evaluate +import evaluate as hf_evaluate from loguru import logger as eval_logger +from lmms_eval.api.model import lmms + MODEL_REGISTRY = {} diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index 310b4bfb3..9c3d57e9f 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -1,45 +1,38 @@ import abc import ast +import inspect import itertools import json - import os -import re import random +import re import shutil -import inspect import subprocess from collections.abc import Callable -from dataclasses import dataclass, field, asdict +from dataclasses import asdict, dataclass, field from glob import glob from typing import Any, List, Union import datasets import numpy as np -from PIL import ImageFile +from accelerate import Accelerator from datasets import DownloadConfig, Image, Sequence from huggingface_hub import snapshot_download -from tenacity import retry, stop_after_attempt, wait_fixed, stop_after_delay +from loguru import logger as eval_logger +from PIL import ImageFile +from tenacity import retry, stop_after_attempt, stop_after_delay, wait_fixed from tqdm import tqdm -from accelerate import Accelerator from lmms_eval import utils from lmms_eval.api import samplers from lmms_eval.api.instance import Instance -from lmms_eval.api.registry import ( - AGGREGATION_REGISTRY, - DEFAULT_METRIC_REGISTRY, - METRIC_REGISTRY, - OUTPUT_TYPE_REGISTRY, - get_aggregation, - get_metric, - get_metric_aggregation, - is_higher_better, -) +from lmms_eval.api.registry import (AGGREGATION_REGISTRY, + DEFAULT_METRIC_REGISTRY, METRIC_REGISTRY, + OUTPUT_TYPE_REGISTRY, get_aggregation, + get_metric, get_metric_aggregation, + is_higher_better) from lmms_eval.filters import build_filter_ensemble -from loguru import logger as eval_logger - # HuggingfaceM4/NoCaps contains truncated image in test split # Include this inside code block to avoid error ImageFile.LOAD_TRUNCATED_IMAGES = True @@ -791,8 +784,8 @@ def _download_from_youtube(path): tar_files = glob(os.path.join(cache_path, "**/*.tar*"), recursive=True) def unzip_video_data(zip_file): - import zipfile import os + import zipfile with zipfile.ZipFile(zip_file, "r") as zip_ref: for file_info in zip_ref.infolist(): diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py index 0e890533f..35f7b7acd 100755 --- a/lmms_eval/evaluator.py +++ b/lmms_eval/evaluator.py @@ -1,34 +1,26 @@ -import os -import time -import random +import collections +import inspect import itertools import json -import collections +import os +import random import sys -import inspect -from tqdm import tqdm - -import torch +import time import numpy as np +import torch from datasets import Image, Sequence +from loguru import logger as eval_logger +from tqdm import tqdm import lmms_eval.api -import lmms_eval.tasks -import lmms_eval.models import lmms_eval.api.metrics import lmms_eval.api.registry - -from lmms_eval.utils import ( - positional_deprecated, - run_task_tests, - make_table, - create_iterator, - get_git_commit_hash, - simple_parse_args_string, -) - -from loguru import logger as eval_logger +import lmms_eval.models +import lmms_eval.tasks +from lmms_eval.utils import (create_iterator, get_git_commit_hash, make_table, + positional_deprecated, run_task_tests, + simple_parse_args_string) @positional_deprecated diff --git a/lmms_eval/filters/__init__.py b/lmms_eval/filters/__init__.py index 38ab7d8a5..f6c353f52 100755 --- a/lmms_eval/filters/__init__.py +++ b/lmms_eval/filters/__init__.py @@ -1,8 +1,6 @@ -from lmms_eval.api.filter import FilterEnsemble, Filter -from . import selection -from . import extraction -from . import transformation +from lmms_eval.api.filter import Filter, FilterEnsemble +from . import extraction, selection, transformation FILTER_REGISTRY = { "take_first": selection.TakeFirstFilter, diff --git a/lmms_eval/filters/extraction.py b/lmms_eval/filters/extraction.py index f3045673b..392e21add 100755 --- a/lmms_eval/filters/extraction.py +++ b/lmms_eval/filters/extraction.py @@ -1,6 +1,7 @@ import re import sys import unicodedata + from lmms_eval.api.filter import Filter diff --git a/lmms_eval/logging_utils.py b/lmms_eval/logging_utils.py index 4d7e09109..bccd0322d 100755 --- a/lmms_eval/logging_utils.py +++ b/lmms_eval/logging_utils.py @@ -1,18 +1,19 @@ # Code mostly from: https://github.com/EleutherAI/lm-evaluation-harness/pull/1339, credit to: https://github.com/ayulockin import copy - -import re -import os -import json import glob -import pandas as pd -import numpy as np +import json +import os +import re from datetime import datetime from typing import Any, Dict, List, Literal, Tuple, Union -from packaging.version import Version -from lmms_eval import utils + +import numpy as np +import pandas as pd import tenacity from loguru import logger +from packaging.version import Version + +from lmms_eval import utils try: import wandb diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py index d171f1082..482fafcbd 100755 --- a/lmms_eval/models/__init__.py +++ b/lmms_eval/models/__init__.py @@ -1,9 +1,9 @@ import importlib import os -import hf_transfer -from loguru import logger import sys + import hf_transfer +from loguru import logger os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" diff --git a/lmms_eval/models/batch_gpt4.py b/lmms_eval/models/batch_gpt4.py index 7541b7098..1926a0912 100755 --- a/lmms_eval/models/batch_gpt4.py +++ b/lmms_eval/models/batch_gpt4.py @@ -1,26 +1,25 @@ # Standard library imports -from copy import deepcopy -from io import BytesIO import base64 - +import json import os import time -import json +from copy import deepcopy +from io import BytesIO +import numpy as np +import requests as url_requests # Related third-party imports from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState -import numpy as np +from loguru import logger as eval_logger +from openai import OpenAI from PIL import Image -import requests as url_requests from tqdm import tqdm -from openai import OpenAI # Local application/library specific imports from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from loguru import logger as eval_logger # Conditional imports try: diff --git a/lmms_eval/models/claude.py b/lmms_eval/models/claude.py index 5829fbed4..edb72b0f4 100644 --- a/lmms_eval/models/claude.py +++ b/lmms_eval/models/claude.py @@ -1,20 +1,19 @@ -from io import BytesIO -from copy import deepcopy -import os import base64 import json +import os +import time +from copy import deepcopy +from io import BytesIO from typing import List, Tuple, Union + +from accelerate import Accelerator, DistributedType +from PIL import Image from tqdm import tqdm -import time from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType - -from PIL import Image - NUM_SECONDS_TO_SLEEP = 5 from loguru import logger @@ -23,8 +22,8 @@ try: import anthropic - from decord import VideoReader, cpu import numpy as np + from decord import VideoReader, cpu except Exception as e: eval_logger.warning(f"Error importing claude: {e}") diff --git a/lmms_eval/models/from_log.py b/lmms_eval/models/from_log.py index c774a086c..7de1f4932 100644 --- a/lmms_eval/models/from_log.py +++ b/lmms_eval/models/from_log.py @@ -1,16 +1,16 @@ import json import os import re - from datetime import datetime from typing import List, Tuple -from tqdm import tqdm -from lmms_eval.api.registry import register_model -from lmms_eval.api.model import lmms -from lmms_eval.api.instance import Instance -from accelerate import Accelerator, DistributedType +from accelerate import Accelerator, DistributedType from loguru import logger as eval_logger +from tqdm import tqdm + +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model @register_model("from_log") diff --git a/lmms_eval/models/fuyu.py b/lmms_eval/models/fuyu.py index fa108358e..5be014903 100755 --- a/lmms_eval/models/fuyu.py +++ b/lmms_eval/models/fuyu.py @@ -3,20 +3,21 @@ warnings.simplefilter("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore") -from accelerate import Accelerator, DistributedType -from transformers import FuyuForCausalLM, AutoTokenizer, FuyuImageProcessor, FuyuProcessor -from lmms_eval.api.model import lmms -from lmms_eval.api.registry import register_model +from typing import List, Optional, Tuple, Union + import torch -from PIL import Image -from typing import List, Optional, Union, Tuple -from lmms_eval import utils -from lmms_eval.api.instance import Instance -from tqdm import tqdm from accelerate import Accelerator, DistributedType from accelerate.state import AcceleratorState - from loguru import logger as eval_logger +from PIL import Image +from tqdm import tqdm +from transformers import (AutoTokenizer, FuyuForCausalLM, FuyuImageProcessor, + FuyuProcessor) + +from lmms_eval import utils +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model @register_model("fuyu") diff --git a/lmms_eval/models/gemini_api.py b/lmms_eval/models/gemini_api.py index 4dbc25bd6..7bebf129e 100644 --- a/lmms_eval/models/gemini_api.py +++ b/lmms_eval/models/gemini_api.py @@ -1,22 +1,21 @@ import io +import json import os import time +from typing import List, Tuple -import json - +from accelerate import Accelerator, DistributedType +from loguru import logger as eval_logger from PIL import Image -from typing import List, Tuple from tqdm import tqdm -from lmms_eval.api.registry import register_model -from lmms_eval.api.model import lmms -from lmms_eval.api.instance import Instance -from accelerate import Accelerator, DistributedType -from loguru import logger as eval_logger +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model try: import google.generativeai as genai - from google.generativeai.types import HarmCategory, HarmBlockThreshold + from google.generativeai.types import HarmBlockThreshold, HarmCategory NUM_SECONDS_TO_SLEEP = 30 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") diff --git a/lmms_eval/models/gpt4v.py b/lmms_eval/models/gpt4v.py index 7d9c5850e..8917c5f95 100755 --- a/lmms_eval/models/gpt4v.py +++ b/lmms_eval/models/gpt4v.py @@ -1,20 +1,20 @@ -from io import BytesIO -from copy import deepcopy -import numpy as np -import os import base64 +import json +import os +import time +from copy import deepcopy +from io import BytesIO from typing import List, Tuple -from tqdm import tqdm + +import numpy as np import requests as url_requests -import time -import json +from accelerate import Accelerator, DistributedType +from tqdm import tqdm from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType - try: from decord import VideoReader, cpu except ImportError: diff --git a/lmms_eval/models/idefics2.py b/lmms_eval/models/idefics2.py index 090cdee5f..644ae41e7 100644 --- a/lmms_eval/models/idefics2.py +++ b/lmms_eval/models/idefics2.py @@ -1,16 +1,16 @@ -import torch +import warnings +from typing import List, Optional, Tuple, Union +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState from tqdm import tqdm +from transformers import AutoProcessor, Idefics2ForConditionalGeneration + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType -from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -from transformers import Idefics2ForConditionalGeneration, AutoProcessor - -import warnings warnings.filterwarnings("ignore") diff --git a/lmms_eval/models/instructblip.py b/lmms_eval/models/instructblip.py index c33ab1228..304bea301 100755 --- a/lmms_eval/models/instructblip.py +++ b/lmms_eval/models/instructblip.py @@ -1,23 +1,22 @@ -import torch - import copy +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import transformers +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState from tqdm import tqdm +from transformers import (InstructBlipForConditionalGeneration, + InstructBlipProcessor) + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model from lmms_eval.tasks.mmmu.utils_group_img import process_images -from accelerate import Accelerator, DistributedType -from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -import transformers -from transformers import InstructBlipProcessor, InstructBlipForConditionalGeneration - from lmms_eval.utils import stop_sequences_criteria - -import warnings - warnings.filterwarnings("ignore") from loguru import logger as eval_logger diff --git a/lmms_eval/models/internvl.py b/lmms_eval/models/internvl.py index 6238d7fcf..26b2dfb77 100644 --- a/lmms_eval/models/internvl.py +++ b/lmms_eval/models/internvl.py @@ -1,25 +1,25 @@ +import math import os +import subprocess +from datetime import timedelta +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import numpy as np +import requests +import torch from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -import torch +from huggingface_hub import snapshot_download +from PIL import Image from tqdm import tqdm -import numpy as np -import math -from datetime import timedelta from transformers import AutoConfig -from huggingface_hub import snapshot_download -import requests from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model from lmms_eval.utils import stop_sequences_criteria -from PIL import Image - -import subprocess -from pathlib import Path wd = Path(__file__).parent.parent.parent.resolve() import sys @@ -31,10 +31,13 @@ eval_logger.internvl_warning_logged = False try: - from internvl.model.internlm2.modeling_internlm2 import InternLM2ForCausalLM - from internvl.model.internvl_chat.configuration_internvl_chat import InternVLChatConfig - from internvl.model.internvl_chat.modeling_intern_vit import InternVisionModel + from internvl.model.internlm2.modeling_internlm2 import \ + InternLM2ForCausalLM from internvl.model.internvl_chat import InternVLChatModel + from internvl.model.internvl_chat.configuration_internvl_chat import \ + InternVLChatConfig + from internvl.model.internvl_chat.modeling_intern_vit import \ + InternVisionModel from internvl.train.dataset import build_transform, dynamic_preprocess except ImportError: eval_logger.debug("InternVL is not installed. Please install InternVL to use this model.") @@ -42,20 +45,19 @@ eval_logger.debug("InternVL is not installed. Please install InternVL to use this model.") eval_logger.internvl_warning_logged = True +import re import warnings from typing import Any, List, Optional, Tuple, Union import torch.utils.checkpoint - +from huggingface_hub import snapshot_download from peft import LoraConfig, get_peft_model from torch import nn from torch.nn import CrossEntropyLoss -from transformers import AutoModel, GenerationConfig, LlamaForCausalLM, LlamaTokenizer +from transformers import (AutoModel, AutoTokenizer, GenerationConfig, + LlamaForCausalLM, LlamaTokenizer) from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.modeling_utils import PreTrainedModel -from transformers import AutoTokenizer -import re -from huggingface_hub import snapshot_download @register_model("internvl") diff --git a/lmms_eval/models/internvl2.py b/lmms_eval/models/internvl2.py index c32258422..3d66c0bfc 100644 --- a/lmms_eval/models/internvl2.py +++ b/lmms_eval/models/internvl2.py @@ -1,17 +1,19 @@ +import logging from typing import List, Tuple -from lmms_eval.api.instance import Instance -from decord import VideoReader, cpu + +import numpy as np import torch import torchvision.transforms as T +from accelerate import Accelerator, DistributedType +from decord import VideoReader, cpu from PIL import Image from torchvision.transforms.functional import InterpolationMode -import numpy as np +from tqdm import tqdm from transformers import AutoModel, AutoTokenizer -from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType + +from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms -from tqdm import tqdm -import logging +from lmms_eval.api.registry import register_model eval_logger = logging.getLogger("eval_logger") @@ -118,6 +120,7 @@ def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=3 from datetime import timedelta + from accelerate.state import AcceleratorState from accelerate.utils import InitProcessGroupKwargs diff --git a/lmms_eval/models/llama_vid.py b/lmms_eval/models/llama_vid.py index e62b4ef71..d3463d88c 100644 --- a/lmms_eval/models/llama_vid.py +++ b/lmms_eval/models/llama_vid.py @@ -1,33 +1,36 @@ +import math import os +import subprocess +from datetime import timedelta +from typing import List, Optional, Tuple, Union + +import numpy as np +import requests +import torch from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -import torch -from tqdm import tqdm from decord import VideoReader, cpu -import numpy as np -import math -from datetime import timedelta -from transformers import AutoConfig from huggingface_hub import snapshot_download -import requests +from loguru import logger as eval_logger +from tqdm import tqdm +from transformers import AutoConfig from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from lmms_eval.utils import stop_sequences_criteria from lmms_eval.models.model_utils.load_video import read_video_pyav - -import subprocess - -from loguru import logger as eval_logger +from lmms_eval.utils import stop_sequences_criteria try: - from llamavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN - from llamavid.conversation import conv_templates, SeparatorStyle + from llamavid.constants import (DEFAULT_IM_END_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX) + from llamavid.conversation import SeparatorStyle, conv_templates from llamavid.model.builder import load_pretrained_model - from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria + from llava.mm_utils import (KeywordsStoppingCriteria, + get_model_name_from_path, + tokenizer_image_token) except ImportError: eval_logger.debug("LLaMA-Video is not installed. Please install LLaMA-Video to use this model.") diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py index 7d6420ba6..1986d0af4 100755 --- a/lmms_eval/models/llava.py +++ b/lmms_eval/models/llava.py @@ -4,8 +4,14 @@ import copy -from tqdm import tqdm +import warnings from datetime import timedelta +from typing import List, Optional, Tuple, Union + +from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs +from accelerate.state import AcceleratorState +from packaging import version +from tqdm import tqdm from lmms_eval import utils from lmms_eval.api.instance import Instance @@ -13,21 +19,16 @@ from lmms_eval.api.registry import register_model from lmms_eval.utils import stop_sequences_criteria -from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs -from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -from packaging import version -import warnings - warnings.filterwarnings("ignore") from loguru import logger as eval_logger try: - from llava.model.builder import load_pretrained_model - from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token - from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN + from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX from llava.conversation import conv_templates + from llava.mm_utils import (get_model_name_from_path, process_images, + tokenizer_image_token) + from llava.model.builder import load_pretrained_model except Exception as e: eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e) diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py index 547daa293..2bce173b5 100644 --- a/lmms_eval/models/llava_hf.py +++ b/lmms_eval/models/llava_hf.py @@ -1,16 +1,17 @@ -import torch +import warnings +from typing import List, Optional, Tuple, Union +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState from tqdm import tqdm +from transformers import (AutoProcessor, LlavaForConditionalGeneration, + LlavaNextForConditionalGeneration) + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType -from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -from transformers import LlavaForConditionalGeneration, LlavaNextForConditionalGeneration, AutoProcessor - -import warnings warnings.filterwarnings("ignore") diff --git a/lmms_eval/models/llava_onevision.py b/lmms_eval/models/llava_onevision.py index 3e7682257..83ae24294 100644 --- a/lmms_eval/models/llava_onevision.py +++ b/lmms_eval/models/llava_onevision.py @@ -1,22 +1,21 @@ -import math -import re import copy import json import logging +import math +import re import warnings from datetime import timedelta -from typing import List, Optional, Union, Tuple -import PIL +from typing import List, Optional, Tuple, Union import numpy as np +import PIL import torch import transformers -from tqdm import tqdm -from packaging import version -from decord import VideoReader, cpu - from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState +from decord import VideoReader, cpu +from packaging import version +from tqdm import tqdm from transformers import AutoConfig from lmms_eval import utils @@ -36,21 +35,14 @@ # Import LLaVA modules try: + from llava.constants import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, + DEFAULT_IMAGE_TOKEN, IGNORE_INDEX, + IMAGE_TOKEN_INDEX) + from llava.conversation import SeparatorStyle, conv_templates + from llava.mm_utils import (KeywordsStoppingCriteria, + get_model_name_from_path, process_images, + tokenizer_image_token) from llava.model.builder import load_pretrained_model - from llava.mm_utils import ( - get_model_name_from_path, - process_images, - tokenizer_image_token, - KeywordsStoppingCriteria, - ) - from llava.constants import ( - IMAGE_TOKEN_INDEX, - DEFAULT_IMAGE_TOKEN, - DEFAULT_IM_START_TOKEN, - DEFAULT_IM_END_TOKEN, - IGNORE_INDEX, - ) - from llava.conversation import conv_templates, SeparatorStyle except ImportError as e: eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}") @@ -454,17 +446,7 @@ def _collate(x): self._config.image_aspect_ratio = getattr(gen_kwargs, "image_aspect_ratio", "pad") eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}") - if type(visual[0]) == PIL.Image.Image and "task_type" not in metadata and "sample_frames" not in metadata: # For image task - image_tensor = process_images(visual, self._image_processor, self._config) - if type(image_tensor) is list: - image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor] - else: - image_tensor = image_tensor.to(dtype=torch.float16, device=self.device) - - task_type = "image" - placeholder_count = len(visual) if isinstance(visual, list) else 1 - - elif "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: + if "task_type" in metadata and metadata["task_type"] == "video" and "sample_frames" in metadata: # overwrite logic for video task assert type(visual) == list, "sample_frames must be specified for video task" sample_indices = np.linspace(0, len(visual) - 1, metadata["sample_frames"], dtype=int) visual = [visual[i] for i in sample_indices] @@ -479,21 +461,38 @@ def _collate(x): task_type = "video" placeholder_count = 1 - elif type(visual[0]) == str: # For video task - image_tensor = [] - try: - if self.video_decode_backend == "decord": - frames = self.load_video(visual, self.max_frames_num) - elif self.video_decode_backend == "pyav": - frames = read_video_pyav(visual[0], num_frm=self.max_frames_num) - frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda() - image_tensor.append(frames) - except Exception as e: - eval_logger.error(f"Error {e} in loading video") + else: # default logic for image and text tasks + if visual == [] and "task_type" not in metadata and "sample_frames" not in metadata: + visual = None + task_type = "text" + placeholder_count = 0 image_tensor = None - task_type = "video" - placeholder_count = len(frames) if self.token_strategy == "multiple" else 1 + elif type(visual[0]) == PIL.Image.Image and "task_type" not in metadata and "sample_frames" not in metadata: # For image task + image_tensor = process_images(visual, self._image_processor, self._config) + if type(image_tensor) is list: + image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor] + else: + image_tensor = image_tensor.to(dtype=torch.float16, device=self.device) + + task_type = "image" + placeholder_count = len(visual) if isinstance(visual, list) else 1 + + elif type(visual[0]) == str: # For video task + image_tensor = [] + try: + if self.video_decode_backend == "decord": + frames = self.load_video(visual, self.max_frames_num) + elif self.video_decode_backend == "pyav": + frames = read_video_pyav(visual[0], num_frm=self.max_frames_num) + frames = self._image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().cuda() + image_tensor.append(frames) + except Exception as e: + eval_logger.error(f"Error {e} in loading video") + image_tensor = None + + task_type = "video" + placeholder_count = len(frames) if self.token_strategy == "multiple" else 1 if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context: """ @@ -553,30 +552,6 @@ def _collate(x): input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device) attention_masks = input_ids.ne(pad_token_ids).to(self.device) - # message_stage = [ - # { - # "from": "human", - # "value": question, - # }, - # { - # "from": "gpt", - # "value": None, - # } - # ] - # qwen_input_ids = self.preprocess_qwen(message_stage, self.tokenizer, has_image=True).to(self.device) - # qwen_result_list = qwen_input_ids.detach().cpu().numpy().tolist() - # qwen_result_list = [i if i != -200 else 100 for i in qwen_result_list[0]] - # qwen_input_text = self.tokenizer.decode(qwen_result_list) - - # original_result_list = input_ids.detach().cpu().numpy().tolist() - # original_result_list = [i if i != -200 else 100 for i in original_result_list[0]] - # original_input_text = self.tokenizer.decode(original_result_list) - - # print(f"Qwen input text: {qwen_input_text}") - # print(f"Original input text: {original_input_text}") - - # assert qwen_input_ids == input_ids - if task_type == "image": gen_kwargs["image_sizes"] = [batched_visuals[0][idx].size for idx in range(len(batched_visuals[0]))] elif task_type == "video": diff --git a/lmms_eval/models/llava_sglang.py b/lmms_eval/models/llava_sglang.py index d473e4c11..233bbe9c4 100644 --- a/lmms_eval/models/llava_sglang.py +++ b/lmms_eval/models/llava_sglang.py @@ -1,23 +1,24 @@ -import torch import random +import torch + torch.backends.cuda.matmul.allow_tf32 = True -from tqdm import tqdm +import warnings from datetime import timedelta +from typing import List, Optional, Tuple, Union + +from tqdm import tqdm from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from typing import List, Optional, Union, Tuple -import warnings - warnings.filterwarnings("ignore") -from concurrent.futures import ThreadPoolExecutor, as_completed import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed from loguru import logger as eval_logger diff --git a/lmms_eval/models/llava_vid.py b/lmms_eval/models/llava_vid.py index c99d1576b..094aee10d 100755 --- a/lmms_eval/models/llava_vid.py +++ b/lmms_eval/models/llava_vid.py @@ -1,28 +1,33 @@ +import copy +import math +from datetime import timedelta +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -import torch -from tqdm import tqdm from decord import VideoReader, cpu -import numpy as np -import math -from datetime import timedelta +from loguru import logger as eval_logger +from tqdm import tqdm from transformers import AutoConfig -import copy from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model from lmms_eval.models.model_utils.load_video import read_video_pyav -from loguru import logger as eval_logger - try: + from llavavid.constants import (DEFAULT_IM_END_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IMAGE_TOKEN, IGNORE_INDEX, + IMAGE_TOKEN_INDEX) + from llavavid.conversation import SeparatorStyle, conv_templates + from llavavid.mm_utils import (KeywordsStoppingCriteria, + get_model_name_from_path, preprocess_llama3, + preprocess_qwen, tokenizer_image_token, + tokenizer_image_token_qwen_merge) from llavavid.model.builder import load_pretrained_model - from llavavid.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria - from llavavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX - from llavavid.conversation import conv_templates, SeparatorStyle - from llavavid.mm_utils import tokenizer_image_token_qwen_merge, preprocess_qwen, preprocess_llama3 except ImportError: eval_logger.debug("LLaVA-Video is not installed. Please install LLaVA-Video to use this model.") @@ -124,8 +129,9 @@ def __init__( overwrite_config["tokenizer_model_max_length"] = 4096 * scaling_factor if "v1.5" in pretrained: # A hardcode solution here to load v1.5 model, otherwise it will use LlavaConfig from hf transformers + from llavavid.model.language_model.llava_llama import ( + LlavaConfig, LlavaLlamaForCausalLM) from transformers import AutoTokenizer - from llavavid.model.language_model.llava_llama import LlavaConfig, LlavaLlamaForCausalLM self._tokenizer = AutoTokenizer.from_pretrained(pretrained, use_fast=False) cfg_pretrained = LlavaConfig.from_pretrained(pretrained) diff --git a/lmms_eval/models/longva.py b/lmms_eval/models/longva.py index c5bf6861a..6d9789ae9 100644 --- a/lmms_eval/models/longva.py +++ b/lmms_eval/models/longva.py @@ -1,23 +1,23 @@ +import math + +import torch from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState from transformers import AutoConfig -import math -import torch - torch.backends.cuda.matmul.allow_tf32 = True -from tqdm import tqdm +import copy +import logging +import warnings from datetime import timedelta -from decord import VideoReader, cpu -import numpy as np +from typing import List, Optional, Tuple, Union -import copy +import numpy as np import PIL -from typing import List, Optional, Union, Tuple +from decord import VideoReader, cpu from packaging import version -import warnings -import logging +from tqdm import tqdm warnings.filterwarnings("ignore") @@ -30,10 +30,14 @@ from lmms_eval.models.model_utils.load_video import read_video_pyav try: + from longva.constants import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, + DEFAULT_IMAGE_TOKEN, IGNORE_INDEX, + IMAGE_TOKEN_INDEX) + from longva.conversation import SeparatorStyle, conv_templates + from longva.mm_utils import (KeywordsStoppingCriteria, + get_model_name_from_path, process_images, + tokenizer_image_token) from longva.model.builder import load_pretrained_model - from longva.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token, KeywordsStoppingCriteria - from longva.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX - from longva.conversation import conv_templates, SeparatorStyle except Exception as e: eval_logger.debug("longva is not installed. Please install longva to use this model.\nError: %s" % e) diff --git a/lmms_eval/models/minicpm_v.py b/lmms_eval/models/minicpm_v.py index 9a6e02159..553c0819b 100755 --- a/lmms_eval/models/minicpm_v.py +++ b/lmms_eval/models/minicpm_v.py @@ -1,17 +1,16 @@ -import torch +import warnings +from typing import List, Optional, Tuple, Union +import torch +from accelerate import Accelerator, DistributedType +from accelerate.state import AcceleratorState from tqdm import tqdm +from transformers import AutoModel, AutoTokenizer + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType -from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -from transformers import AutoModel, AutoTokenizer - - -import warnings warnings.filterwarnings("ignore") diff --git a/lmms_eval/models/model_utils/load_video.py b/lmms_eval/models/model_utils/load_video.py index dbb3cf6f4..b44862a7c 100644 --- a/lmms_eval/models/model_utils/load_video.py +++ b/lmms_eval/models/model_utils/load_video.py @@ -1,6 +1,6 @@ import av -from av.codec.context import CodecContext import numpy as np +from av.codec.context import CodecContext # This one is faster diff --git a/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py b/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py index 3717665a0..a11a49fa1 100755 --- a/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py +++ b/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py @@ -9,16 +9,15 @@ warnings.simplefilter("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore") -from typing import Tuple, List, Union, Iterable +from typing import Iterable, List, Tuple, Union import numpy as np import torch import torch.nn.functional as F +from loguru import logger from transformers import PreTrainedTokenizer from transformers.generation import LogitsProcessor -from loguru import logger - # Types. HistoryType = List[Tuple[str, str]] TokensType = List[int] diff --git a/lmms_eval/models/mplug_owl_video.py b/lmms_eval/models/mplug_owl_video.py index 1055f4dd6..3b458cbba 100644 --- a/lmms_eval/models/mplug_owl_video.py +++ b/lmms_eval/models/mplug_owl_video.py @@ -1,20 +1,20 @@ +from datetime import timedelta +from typing import List, Optional, Tuple, Union + +import torch from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -import torch -from transformers import AutoTokenizer +from loguru import logger from tqdm import tqdm -from datetime import timedelta +from transformers import AutoTokenizer from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model - -from lmms_eval.models.mplug_owl_video.modeling_mplug_owl import MplugOwlForConditionalGeneration -from lmms_eval.models.mplug_owl_video.processing_mplug_owl import MplugOwlImageProcessor, MplugOwlProcessor - - -from loguru import logger +from lmms_eval.models.mplug_owl_video.modeling_mplug_owl import \ + MplugOwlForConditionalGeneration +from lmms_eval.models.mplug_owl_video.processing_mplug_owl import ( + MplugOwlImageProcessor, MplugOwlProcessor) eval_logger = logger diff --git a/lmms_eval/models/mplug_owl_video/__init__.py b/lmms_eval/models/mplug_owl_video/__init__.py index 2020ad3a8..0889695b3 100644 --- a/lmms_eval/models/mplug_owl_video/__init__.py +++ b/lmms_eval/models/mplug_owl_video/__init__.py @@ -13,8 +13,8 @@ # limitations under the License. from typing import TYPE_CHECKING -from transformers.utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available - +from transformers.utils import (OptionalDependencyNotAvailable, _LazyModule, + is_tokenizers_available, is_torch_available) _import_structure = { "configuration_mplug_owl": ["MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP", "MplugOwlConfig"], @@ -43,7 +43,8 @@ if TYPE_CHECKING: - from .configuration_mplug_owl import MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP, MplugOwlConfig + from .configuration_mplug_owl import ( + MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP, MplugOwlConfig) from .tokenization_mplug_owl import MplugOwlTokenizer try: @@ -60,10 +61,8 @@ else: from .modeling_mplug_owl import ( MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST, - MplugOwlForConditionalGeneration, - MplugOwlModel, - MplugOwlPreTrainedModel, - ) + MplugOwlForConditionalGeneration, MplugOwlModel, + MplugOwlPreTrainedModel) else: diff --git a/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py b/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py index ce5b777bb..89edcbd0e 100644 --- a/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py +++ b/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py @@ -17,11 +17,11 @@ import os from typing import Union +from loguru import logger from transformers.configuration_utils import PretrainedConfig -from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES from transformers.models.auto import CONFIG_MAPPING - -from loguru import logger +from transformers.models.auto.modeling_auto import \ + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP = { "MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/config.json", diff --git a/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py b/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py index 07bf9f0cc..a698d0c02 100644 --- a/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py +++ b/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py @@ -16,6 +16,7 @@ import math from typing import Any, Optional, Tuple, Union + from loguru import logger try: @@ -30,22 +31,23 @@ from dataclasses import dataclass from typing import Any, Optional, Tuple, Union +import einops import torch import torch.utils.checkpoint from torch import nn -import einops - -from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, BaseModelOutputWithPastAndCrossAttentions +from transformers.modeling_outputs import ( + BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling) from transformers.modeling_utils import PreTrainedModel -from transformers.pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from transformers.utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) from transformers.models.auto import AutoModelForCausalLM -from .configuration_mplug_owl import MplugOwlConfig, MplugOwlVisionConfig, MplugOwlVisualAbstractorConfig +from transformers.pytorch_utils import (find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.utils import (ModelOutput, add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings) + +from .configuration_mplug_owl import (MplugOwlConfig, MplugOwlVisionConfig, + MplugOwlVisualAbstractorConfig) _CHECKPOINT_FOR_DOC = "MAGAer13/mplug-owl-llama-7b" _CONFIG_FOR_DOC = "MplugOwlConfig" diff --git a/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py b/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py index 38cbf023f..705404155 100644 --- a/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py +++ b/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py @@ -1,17 +1,18 @@ import re + +import numpy as np import torch import torch.utils.checkpoint - +from decord import VideoReader +from PIL import Image +from transformers.models.clip.image_processing_clip import CLIPImageProcessor from transformers.processing_utils import ProcessorMixin from transformers.tokenization_utils_base import BatchEncoding -from transformers.models.clip.image_processing_clip import CLIPImageProcessor -from .tokenization_mplug_owl import MplugOwlTokenizer -from decord import VideoReader -import numpy as np -from PIL import Image from lmms_eval.models.model_utils.load_video import read_video_pyav +from .tokenization_mplug_owl import MplugOwlTokenizer + def get_index(num_frames, num_segments): seg_size = float(num_frames - 1) / num_segments diff --git a/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py b/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py index bd7f42f12..c9325d1de 100644 --- a/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py +++ b/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py @@ -14,9 +14,8 @@ # limitations under the License. """Tokenization classes for MplugOwl.""" -from transformers.models.llama.tokenization_llama import LlamaTokenizer - from loguru import logger +from transformers.models.llama.tokenization_llama import LlamaTokenizer VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"} diff --git a/lmms_eval/models/phi3v.py b/lmms_eval/models/phi3v.py index ab1e838d8..61e36386c 100644 --- a/lmms_eval/models/phi3v.py +++ b/lmms_eval/models/phi3v.py @@ -1,16 +1,15 @@ -import torch +from typing import List, Optional, Tuple, Union +import torch from accelerate import Accelerator, DistributedType +from loguru import logger as eval_logger +from tqdm import tqdm +from transformers import AutoModelForCausalLM, AutoProcessor + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from tqdm import tqdm -from transformers import AutoModelForCausalLM -from transformers import AutoProcessor -from typing import List, Optional, Tuple, Union - -from loguru import logger as eval_logger @register_model("phi3v") diff --git a/lmms_eval/models/qwen_vl.py b/lmms_eval/models/qwen_vl.py index dff301d30..79adc5324 100755 --- a/lmms_eval/models/qwen_vl.py +++ b/lmms_eval/models/qwen_vl.py @@ -1,17 +1,17 @@ -import torch +import os +import uuid +import warnings +from typing import List, Optional, Tuple, Union +import torch +from accelerate import Accelerator, DistributedType from tqdm import tqdm + from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model from lmms_eval.models.model_utils.qwen.qwen_generate_utils import make_context -from accelerate import Accelerator, DistributedType -from typing import List, Optional, Union, Tuple -import uuid -import os - -import warnings warnings.simplefilter("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore") diff --git a/lmms_eval/models/qwen_vl_api.py b/lmms_eval/models/qwen_vl_api.py index 8a0ccc653..47a25a153 100644 --- a/lmms_eval/models/qwen_vl_api.py +++ b/lmms_eval/models/qwen_vl_api.py @@ -1,19 +1,18 @@ -from io import BytesIO -from copy import deepcopy -import os import base64 -from typing import List, Tuple, Union -from tqdm import tqdm -import requests as url_requests +import os import time +from copy import deepcopy +from io import BytesIO +from typing import List, Tuple, Union +import requests as url_requests +from PIL import Image +from tqdm import tqdm +from lmms_eval import utils from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from lmms_eval import utils - -from PIL import Image NUM_SECONDS_TO_SLEEP = 5 from loguru import logger as eval_logger diff --git a/lmms_eval/models/reka.py b/lmms_eval/models/reka.py index ee1b9c670..744f54ee9 100644 --- a/lmms_eval/models/reka.py +++ b/lmms_eval/models/reka.py @@ -1,20 +1,20 @@ -from PIL import Image -from io import BytesIO -from copy import deepcopy -import numpy as np -import os import base64 -from typing import List, Tuple -from tqdm import tqdm -import requests as url_requests +import json +import os import time +from copy import deepcopy +from io import BytesIO +from typing import List, Tuple -import json +import numpy as np +import requests as url_requests +from accelerate import Accelerator, DistributedType +from PIL import Image +from tqdm import tqdm from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from accelerate import Accelerator, DistributedType NUM_SECONDS_TO_SLEEP = 30 @@ -23,9 +23,9 @@ eval_logger = logger try: - from reka.client import Reka as RekaClient - from reka import ChatMessage from decord import VideoReader, cpu + from reka import ChatMessage + from reka.client import Reka as RekaClient except Exception as e: eval_logger.warning(f"Error importing reka: {e}") diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py index ca9d9e900..18cb1625d 100755 --- a/lmms_eval/models/srt_api.py +++ b/lmms_eval/models/srt_api.py @@ -1,23 +1,23 @@ -from accelerate import Accelerator, DistributedType import base64 -from io import BytesIO +import json +import os +import time from copy import deepcopy -from decord import VideoReader, cpu +from io import BytesIO +from typing import List, Tuple + import numpy as np +from accelerate import Accelerator, DistributedType +from decord import VideoReader, cpu +from loguru import logger as eval_logger from openai import OpenAI from PIL import Image -import os -import json -from typing import List, Tuple from tqdm import tqdm -import time from lmms_eval.api.instance import Instance from lmms_eval.api.model import lmms from lmms_eval.api.registry import register_model -from loguru import logger as eval_logger - NUM_SECONDS_TO_SLEEP = 5 @@ -177,22 +177,6 @@ def generate_until(self, requests) -> List[str]: content.append({"type": "text", "text": contexts}) messages.append({"role": "user", "content": content}) - # if self.image_token not in contexts: # single image format - # content = [] - # for img in imgs: - # content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}) - - # content.append({"type": "text", "text": contexts}) - # messages.append({"role": "user", "content": content}) - # else: # interleaved format - # contexts = contexts.split(self.image_token) - # for idx, img in enumerate(imgs): - # content = [ - # {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}, - # {"type": "text", "text": contexts[idx]}, - # ] - # messages.append({"role": "user", "content": content}) - # messages.append({"role": "user", "content": [{"type": "text", "text": contexts[-1]}]}) if "max_new_tokens" not in gen_kwargs: gen_kwargs["max_new_tokens"] = 1024 diff --git a/lmms_eval/models/tinyllava.py b/lmms_eval/models/tinyllava.py index a4335f054..3b0ff7cbf 100755 --- a/lmms_eval/models/tinyllava.py +++ b/lmms_eval/models/tinyllava.py @@ -3,8 +3,14 @@ torch.backends.cuda.matmul.allow_tf32 = True import copy -from tqdm import tqdm +import warnings from datetime import timedelta +from typing import List, Optional, Tuple, Union + +from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs +from accelerate.state import AcceleratorState +from packaging import version +from tqdm import tqdm from lmms_eval import utils from lmms_eval.api.instance import Instance @@ -12,19 +18,13 @@ from lmms_eval.api.registry import register_model from lmms_eval.utils import stop_sequences_criteria -from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs -from accelerate.state import AcceleratorState -from typing import List, Optional, Union, Tuple -from packaging import version -import warnings - warnings.filterwarnings("ignore") from loguru import logger as eval_logger try: - from tinyllava.model import load_pretrained_model from tinyllava.data import ImagePreprocess, TextPreprocess + from tinyllava.model import load_pretrained_model from tinyllava.utils.constants import DEFAULT_IMAGE_TOKEN from tinyllava.utils.message import Message except Exception as e: diff --git a/lmms_eval/models/video_chatgpt.py b/lmms_eval/models/video_chatgpt.py index 089a2c5df..6bc865a65 100644 --- a/lmms_eval/models/video_chatgpt.py +++ b/lmms_eval/models/video_chatgpt.py @@ -1,25 +1,27 @@ import os -from lmms_eval.api.instance import Instance -from lmms_eval.api.model import lmms -from lmms_eval.api.registry import register_model +from datetime import timedelta +from typing import List, Optional, Tuple, Union +import torch from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs from accelerate.state import AcceleratorState from huggingface_hub import snapshot_download -import torch +from loguru import logger from PIL import Image - -from datetime import timedelta -from typing import List, Tuple, Optional, Union from tqdm import tqdm -from loguru import logger +from lmms_eval.api.instance import Instance +from lmms_eval.api.model import lmms +from lmms_eval.api.registry import register_model eval_logger = logger try: - from lmms_eval.models.video_chatgpt.eval.model_utils import load_video, initialize_model - from lmms_eval.models.video_chatgpt.inference import video_chatgpt_infer, video_chatgpt_infer_ppl, get_spatio_temporal_features_torch + from lmms_eval.models.video_chatgpt.eval.model_utils import ( + initialize_model, load_video) + from lmms_eval.models.video_chatgpt.inference import ( + get_spatio_temporal_features_torch, video_chatgpt_infer, + video_chatgpt_infer_ppl) except ImportError: eval_logger.warning("Failed to import video_chatgpt modules") diff --git a/lmms_eval/models/video_chatgpt/eval/model_utils.py b/lmms_eval/models/video_chatgpt/eval/model_utils.py index fc9162400..8d95bd150 100644 --- a/lmms_eval/models/video_chatgpt/eval/model_utils.py +++ b/lmms_eval/models/video_chatgpt/eval/model_utils.py @@ -1,12 +1,14 @@ import os + import numpy as np -from PIL import Image +import torch from decord import VideoReader, cpu -from transformers import AutoTokenizer, CLIPVisionModel, CLIPImageProcessor +from PIL import Image +from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel + +from lmms_eval.models.video_chatgpt.constants import * from lmms_eval.models.video_chatgpt.model import VideoChatGPTLlamaForCausalLM from lmms_eval.models.video_chatgpt.utils import disable_torch_init -from lmms_eval.models.video_chatgpt.constants import * -import torch def load_video(vis_path, n_clips=1, num_frm=100): diff --git a/lmms_eval/models/video_chatgpt/inference.py b/lmms_eval/models/video_chatgpt/inference.py index b76695257..3ae126dd1 100644 --- a/lmms_eval/models/video_chatgpt/inference.py +++ b/lmms_eval/models/video_chatgpt/inference.py @@ -1,7 +1,9 @@ -from lmms_eval.models.video_chatgpt.video_conversation import conv_templates, SeparatorStyle -from lmms_eval.models.video_chatgpt.model.utils import KeywordsStoppingCriteria import torch +from lmms_eval.models.video_chatgpt.model.utils import KeywordsStoppingCriteria +from lmms_eval.models.video_chatgpt.video_conversation import (SeparatorStyle, + conv_templates) + # Define constants DEFAULT_VIDEO_TOKEN = "