cbrkit.eval.retrieval 

 1from collections.abc import Sequence
 2from typing import Any, Literal
 3
 4from ..helpers import unpack_float
 5from ..retrieval import Result, ResultStep
 6from ..typing import EvalMetricFunc, Float, QueryCaseMatrix
 7from .common import DEFAULT_METRICS, compute, similarities_to_qrels
 8
 9
10def retrieval_step[Q, C, S: Float](
11    qrels: QueryCaseMatrix[Q, C, int],
12    step: ResultStep[Q, C, Any, S],
13    metrics: Sequence[str] = DEFAULT_METRICS,
14    metric_funcs: dict[str, EvalMetricFunc] | None = None,
15) -> dict[str, float]:
16    return compute(
17        qrels,
18        {query: entry.similarities for query, entry in step.queries.items()},
19        metrics,
20        metric_funcs,
21    )
22
23
24def retrieval[Q, C, S: Float](
25    qrels: QueryCaseMatrix[Q, C, int],
26    result: Result[Q, C, Any, S],
27    metrics: Sequence[str] = DEFAULT_METRICS,
28    metric_funcs: dict[str, EvalMetricFunc] | None = None,
29) -> list[dict[str, float]]:
30    return [
31        retrieval_step(
32            qrels,
33            step,
34            metrics,
35            metric_funcs,
36        )
37        for step in result.steps
38    ]
39
40
41def retrieval_step_to_qrels[Q, C, S: Float](
42    result: ResultStep[Q, C, Any, S],
43    max_qrel: int | None = None,
44    min_qrel: int = 1,
45    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
46    auto_scale: bool = True,
47) -> QueryCaseMatrix[Q, C, int]:
48    unpacked_sims = {
49        query: {case: unpack_float(value) for case, value in entry.similarities.items()}
50        for query, entry in result.queries.items()
51    }
52    return similarities_to_qrels(
53        unpacked_sims,
54        max_qrel,
55        min_qrel,
56        round_mode,
57        auto_scale,
58    )
59
60
61def retrieval_to_qrels[Q, C, S: Float](
62    result: Result[Q, C, Any, S],
63    max_qrel: int = 5,
64    min_qrel: int = 1,
65    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
66    auto_scale: bool = True,
67) -> list[QueryCaseMatrix[Q, C, int]]:
68    return [
69        retrieval_step_to_qrels(
70            step,
71            max_qrel,
72            min_qrel,
73            round_mode,
74            auto_scale,
75        )
76        for step in result.steps
77    ]
def retrieval_step( qrels: QueryCaseMatrix[Q, C, int], step: cbrkit.model.ResultStep[TypeVar, TypeVar, Any, TypeVar], metrics: Sequence[str] = ('precision', 'recall', 'f1', 'map', 'ndcg', 'correctness', 'completeness'), metric_funcs: dict[str, cbrkit.typing.EvalMetricFunc] | None = None) -> dict[str, float]: 
11def retrieval_step[Q, C, S: Float](
12    qrels: QueryCaseMatrix[Q, C, int],
13    step: ResultStep[Q, C, Any, S],
14    metrics: Sequence[str] = DEFAULT_METRICS,
15    metric_funcs: dict[str, EvalMetricFunc] | None = None,
16) -> dict[str, float]:
17    return compute(
18        qrels,
19        {query: entry.similarities for query, entry in step.queries.items()},
20        metrics,
21        metric_funcs,
22    )
def retrieval( qrels: QueryCaseMatrix[Q, C, int], result: cbrkit.model.Result[TypeVar, TypeVar, Any, TypeVar], metrics: Sequence[str] = ('precision', 'recall', 'f1', 'map', 'ndcg', 'correctness', 'completeness'), metric_funcs: dict[str, cbrkit.typing.EvalMetricFunc] | None = None) -> list[dict[str, float]]: 
25def retrieval[Q, C, S: Float](
26    qrels: QueryCaseMatrix[Q, C, int],
27    result: Result[Q, C, Any, S],
28    metrics: Sequence[str] = DEFAULT_METRICS,
29    metric_funcs: dict[str, EvalMetricFunc] | None = None,
30) -> list[dict[str, float]]:
31    return [
32        retrieval_step(
33            qrels,
34            step,
35            metrics,
36            metric_funcs,
37        )
38        for step in result.steps
39    ]
def retrieval_step_to_qrels( result: cbrkit.model.ResultStep[TypeVar, TypeVar, Any, TypeVar], max_qrel: int | None = None, min_qrel: int = 1, round_mode: Literal['floor', 'ceil', 'nearest'] = 'nearest', auto_scale: bool = True) -> QueryCaseMatrix[Q, C, int]: 
42def retrieval_step_to_qrels[Q, C, S: Float](
43    result: ResultStep[Q, C, Any, S],
44    max_qrel: int | None = None,
45    min_qrel: int = 1,
46    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
47    auto_scale: bool = True,
48) -> QueryCaseMatrix[Q, C, int]:
49    unpacked_sims = {
50        query: {case: unpack_float(value) for case, value in entry.similarities.items()}
51        for query, entry in result.queries.items()
52    }
53    return similarities_to_qrels(
54        unpacked_sims,
55        max_qrel,
56        min_qrel,
57        round_mode,
58        auto_scale,
59    )
def retrieval_to_qrels( result: cbrkit.model.Result[TypeVar, TypeVar, Any, TypeVar], max_qrel: int = 5, min_qrel: int = 1, round_mode: Literal['floor', 'ceil', 'nearest'] = 'nearest', auto_scale: bool = True) -> list[QueryCaseMatrix[Q, C, int]]: 
62def retrieval_to_qrels[Q, C, S: Float](
63    result: Result[Q, C, Any, S],
64    max_qrel: int = 5,
65    min_qrel: int = 1,
66    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
67    auto_scale: bool = True,
68) -> list[QueryCaseMatrix[Q, C, int]]:
69    return [
70        retrieval_step_to_qrels(
71            step,
72            max_qrel,
73            min_qrel,
74            round_mode,
75            auto_scale,
76        )
77        for step in result.steps
78    ]