cbrkit.eval.retrieval

View Source

 1from collections.abc import Sequence
 2from typing import Any, Literal
 3
 4from ..helpers import normalize_and_scale, round, unpack_float
 5from ..retrieval import Result, ResultStep
 6from ..typing import EvalMetricFunc, Float, QueryCaseMatrix
 7from .common import DEFAULT_METRICS, compute
 8
 9
10def retrieval_step[Q, C, S: Float](
11    qrels: QueryCaseMatrix[Q, C, int],
12    step: ResultStep[Q, C, Any, S],
13    metrics: Sequence[str] = DEFAULT_METRICS,
14    metric_funcs: dict[str, EvalMetricFunc] | None = None,
15) -> dict[str, float]:
16    return compute(
17        qrels,
18        {query: entry.similarities for query, entry in step.queries.items()},
19        metrics,
20        metric_funcs,
21    )
22
23
24def retrieval[Q, C, S: Float](
25    qrels: QueryCaseMatrix[Q, C, int],
26    result: Result[Q, C, Any, S],
27    metrics: Sequence[str] = DEFAULT_METRICS,
28    metric_funcs: dict[str, EvalMetricFunc] | None = None,
29) -> list[dict[str, float]]:
30    return [
31        retrieval_step(
32            qrels,
33            step,
34            metrics,
35            metric_funcs,
36        )
37        for step in result.steps
38    ]
39
40
41def retrieval_step_to_qrels[Q, C, S: Float](
42    result: ResultStep[Q, C, Any, S],
43    max_qrel: int | None = None,
44    min_qrel: int = 1,
45    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
46    auto_scale: bool = True,
47) -> QueryCaseMatrix[Q, C, int]:
48    if max_qrel is None:
49        return {
50            query: {
51                case: rank
52                for rank, case in enumerate(reversed(entry.ranking), start=min_qrel)
53            }
54            for query, entry in result.queries.items()
55        }
56
57    sims = {
58        query: {case: unpack_float(value) for case, value in entry.similarities.items()}
59        for query, entry in result.queries.items()
60    }
61    if auto_scale:
62        min_sim = min(min(entries.values()) for entries in sims.values())
63        max_sim = max(max(entries.values()) for entries in sims.values())
64    else:
65        min_sim = 0.0
66        max_sim = 1.0
67
68    return {
69        query: {
70            case: round(
71                normalize_and_scale(sim, min_sim, max_sim, min_qrel, max_qrel),
72                round_mode,
73            )
74            for case, sim in entry.items()
75        }
76        for query, entry in sims.items()
77    }
78
79
80def retrieval_to_qrels[Q, C, S: Float](
81    result: Result[Q, C, Any, S],
82    max_qrel: int = 5,
83    min_qrel: int = 1,
84    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
85    auto_scale: bool = True,
86) -> list[QueryCaseMatrix[Q, C, int]]:
87    return [
88        retrieval_step_to_qrels(
89            step,
90            max_qrel,
91            min_qrel,
92            round_mode,
93            auto_scale,
94        )
95        for step in result.steps
96    ]

def retrieval_step( qrels: QueryCaseMatrix[Q, C, int], step: cbrkit.model.ResultStep[TypeVar, TypeVar, Any, TypeVar], metrics: Sequence[str] = ('precision', 'recall', 'f1', 'map', 'ndcg', 'correctness', 'completeness'), metric_funcs: dict[str, cbrkit.typing.EvalMetricFunc] | None = None) -> dict[str, float]: View Source

11def retrieval_step[Q, C, S: Float](
12    qrels: QueryCaseMatrix[Q, C, int],
13    step: ResultStep[Q, C, Any, S],
14    metrics: Sequence[str] = DEFAULT_METRICS,
15    metric_funcs: dict[str, EvalMetricFunc] | None = None,
16) -> dict[str, float]:
17    return compute(
18        qrels,
19        {query: entry.similarities for query, entry in step.queries.items()},
20        metrics,
21        metric_funcs,
22    )

def retrieval( qrels: QueryCaseMatrix[Q, C, int], result: cbrkit.model.Result[TypeVar, TypeVar, Any, TypeVar], metrics: Sequence[str] = ('precision', 'recall', 'f1', 'map', 'ndcg', 'correctness', 'completeness'), metric_funcs: dict[str, cbrkit.typing.EvalMetricFunc] | None = None) -> list[dict[str, float]]: View Source

25def retrieval[Q, C, S: Float](
26    qrels: QueryCaseMatrix[Q, C, int],
27    result: Result[Q, C, Any, S],
28    metrics: Sequence[str] = DEFAULT_METRICS,
29    metric_funcs: dict[str, EvalMetricFunc] | None = None,
30) -> list[dict[str, float]]:
31    return [
32        retrieval_step(
33            qrels,
34            step,
35            metrics,
36            metric_funcs,
37        )
38        for step in result.steps
39    ]

def retrieval_step_to_qrels( result: cbrkit.model.ResultStep[TypeVar, TypeVar, Any, TypeVar], max_qrel: int | None = None, min_qrel: int = 1, round_mode: Literal['floor', 'ceil', 'nearest'] = 'nearest', auto_scale: bool = True) -> QueryCaseMatrix[Q, C, int]: View Source

42def retrieval_step_to_qrels[Q, C, S: Float](
43    result: ResultStep[Q, C, Any, S],
44    max_qrel: int | None = None,
45    min_qrel: int = 1,
46    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
47    auto_scale: bool = True,
48) -> QueryCaseMatrix[Q, C, int]:
49    if max_qrel is None:
50        return {
51            query: {
52                case: rank
53                for rank, case in enumerate(reversed(entry.ranking), start=min_qrel)
54            }
55            for query, entry in result.queries.items()
56        }
57
58    sims = {
59        query: {case: unpack_float(value) for case, value in entry.similarities.items()}
60        for query, entry in result.queries.items()
61    }
62    if auto_scale:
63        min_sim = min(min(entries.values()) for entries in sims.values())
64        max_sim = max(max(entries.values()) for entries in sims.values())
65    else:
66        min_sim = 0.0
67        max_sim = 1.0
68
69    return {
70        query: {
71            case: round(
72                normalize_and_scale(sim, min_sim, max_sim, min_qrel, max_qrel),
73                round_mode,
74            )
75            for case, sim in entry.items()
76        }
77        for query, entry in sims.items()
78    }

def retrieval_to_qrels( result: cbrkit.model.Result[TypeVar, TypeVar, Any, TypeVar], max_qrel: int = 5, min_qrel: int = 1, round_mode: Literal['floor', 'ceil', 'nearest'] = 'nearest', auto_scale: bool = True) -> list[QueryCaseMatrix[Q, C, int]]: View Source

81def retrieval_to_qrels[Q, C, S: Float](
82    result: Result[Q, C, Any, S],
83    max_qrel: int = 5,
84    min_qrel: int = 1,
85    round_mode: Literal["floor", "ceil", "nearest"] = "nearest",
86    auto_scale: bool = True,
87) -> list[QueryCaseMatrix[Q, C, int]]:
88    return [
89        retrieval_step_to_qrels(
90            step,
91            max_qrel,
92            min_qrel,
93            round_mode,
94            auto_scale,
95        )
96        for step in result.steps
97    ]