Skip to content

Commit

Permalink
formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
elisno committed Feb 11, 2025
1 parent 9e91e9b commit c5843c9
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 15 deletions.
13 changes: 8 additions & 5 deletions src/cleanlab_codex/codex_backup.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,14 @@ def run(
response,
query=query,
context=context,
config=cast(BadResponseDetectionConfig, {
"tlm": self._tlm,
"fallback_answer": self._fallback_answer,
**_is_bad_response_kwargs,
}),
config=cast(
BadResponseDetectionConfig,
{
"tlm": self._tlm,
"fallback_answer": self._fallback_answer,
**_is_bad_response_kwargs,
},
),
):
return response

Expand Down
21 changes: 11 additions & 10 deletions src/cleanlab_codex/response_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,18 @@ def get_trustworthiness_score(
prompt: Union[str, Sequence[str]],
response: Union[str, Sequence[str]],
**kwargs: Any,
) -> Dict[str, Any]:
...
) -> Dict[str, Any]: ...

def prompt(
self,
prompt: Union[str, Sequence[str]],
/,
**kwargs: Any,
) -> Dict[str, Any]:
...
) -> Dict[str, Any]: ...

TLM = _TLMProtocol



DEFAULT_FALLBACK_ANSWER = "Based on the available information, I cannot provide a complete answer to this question."
DEFAULT_PARTIAL_RATIO_THRESHOLD = 70
DEFAULT_TRUSTWORTHINESS_THRESHOLD = 0.5
Expand All @@ -53,6 +50,7 @@ class BadResponseDetectionConfig(TypedDict, total=False):
unhelpfulness_confidence_threshold: Optional confidence threshold (0.0-1.0) for unhelpful classification
tlm: TLM model to use for evaluation (required for untrustworthiness and unhelpfulness checks)
"""

# Fallback check config
fallback_answer: str
partial_ratio_threshold: int
Expand All @@ -67,6 +65,7 @@ class BadResponseDetectionConfig(TypedDict, total=False):
# Shared config (for untrustworthiness and unhelpfulness checks)
tlm: Optional[TLM]


def get_bad_response_config() -> BadResponseDetectionConfig:
"""Get the default configuration for bad response detection functions.
Expand Down Expand Up @@ -119,11 +118,13 @@ def is_bad_response(
validation_checks: list[Callable[[], bool]] = []

# All required inputs are available for checking fallback responses
validation_checks.append(lambda: is_fallback_response(
response,
cfg["fallback_answer"],
threshold=cfg["partial_ratio_threshold"],
))
validation_checks.append(
lambda: is_fallback_response(
response,
cfg["fallback_answer"],
threshold=cfg["partial_ratio_threshold"],
)
)

can_run_untrustworthy_check = query is not None and context is not None and cfg["tlm"] is not None
if can_run_untrustworthy_check:
Expand Down

0 comments on commit c5843c9

Please sign in to comment.