From 0962c3221dfb4c76a24bb2561239d26cbb33e5ce Mon Sep 17 00:00:00 2001 From: Yongtae Date: Tue, 21 Nov 2023 14:53:34 +0900 Subject: [PATCH] Fix bug in calculating score when output from LLM is broken --- src/ragas/metrics/_faithfulness.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ragas/metrics/_faithfulness.py b/src/ragas/metrics/_faithfulness.py index 1da97b955..d3f07d1cf 100644 --- a/src/ragas/metrics/_faithfulness.py +++ b/src/ragas/metrics/_faithfulness.py @@ -120,9 +120,12 @@ def _score_batch( ) score = score / len(list_statements[i]) else: - score = max(0, output.count("verdict: no")) / len( - list_statements[i] - ) + if 'verdict: no' in output or 'verdict: yes' in output: + score = max(0, output.count("verdict: no")) / len( + list_statements[i] + ) + else: # output from LLM is broken and we can't evaluate score. + score = 1 scores.append(1 - score)