From 7c9e44c6f1e48a4ae9174a006fd04b41e6a535f4 Mon Sep 17 00:00:00 2001 From: Roni Friedman-Melamed Date: Thu, 7 Nov 2024 14:18:02 +0200 Subject: [PATCH] test in scripts Signed-off-by: Roni Friedman-Melamed --- {tests => scripts}/test_unitxt.py | 0 src/instructlab/eval/mmlu.py | 6 ++---- 2 files changed, 2 insertions(+), 4 deletions(-) rename {tests => scripts}/test_unitxt.py (100%) diff --git a/tests/test_unitxt.py b/scripts/test_unitxt.py similarity index 100% rename from tests/test_unitxt.py rename to scripts/test_unitxt.py diff --git a/src/instructlab/eval/mmlu.py b/src/instructlab/eval/mmlu.py index f290eb5..16776d2 100644 --- a/src/instructlab/eval/mmlu.py +++ b/src/instructlab/eval/mmlu.py @@ -142,7 +142,7 @@ def run(self, server_url: str | None = None) -> tuple: agg_score: float = 0.0 results = self._run_mmlu(server_url) - for task, result in results['results'].items(): + for task, result in results["results"].items(): agg_score += float(result["acc,none"]) individual_scores[task] = { "score": float(result["acc,none"]), @@ -153,9 +153,7 @@ def run(self, server_url: str | None = None) -> tuple: return overall_score, individual_scores - def _run_mmlu( - self, server_url: str | None = None - ) -> dict: + def _run_mmlu(self, server_url: str | None = None) -> dict: if server_url is not None: # Requires lm_eval >= 0.4.4 model_args = f"base_url={server_url}/completions,model={self.model_path},tokenizer_backend=huggingface"