Skip to content

pydantic_evals.reporting

ConfusionMatrix

Bases: BaseModel

A confusion matrix comparing expected vs predicted labels across cases.

Source code in pydantic_evals/pydantic_evals/reporting/analyses.py 
18
19
20
21
22
23
24
25
26
27
class ConfusionMatrix(BaseModel):
    """A confusion matrix comparing expected vs predicted labels across cases."""

    type: Literal['confusion_matrix'] = 'confusion_matrix'
    title: str = 'Confusion Matrix'
    description: str | None = None
    class_labels: list[str]
    """Ordered list of class labels (used for both axes)."""
    matrix: list[list[int]]
    """matrix[expected_idx][predicted_idx] = count of cases."""

class_labels instance-attribute 

class_labels: list[str]

Ordered list of class labels (used for both axes).

matrix instance-attribute 

matrix: list[list[int]]

matrix[expected_idx][predicted_idx] = count of cases.

PrecisionRecall

Bases: BaseModel

Precision-recall curve data across cases.

Source code in pydantic_evals/pydantic_evals/reporting/analyses.py 
49
50
51
52
53
54
55
56
class PrecisionRecall(BaseModel):
    """Precision-recall curve data across cases."""

    type: Literal['precision_recall'] = 'precision_recall'
    title: str = 'Precision-Recall Curve'
    description: str | None = None
    curves: list[PrecisionRecallCurve]
    """One or more curves."""

curves instance-attribute 

curves: list[PrecisionRecallCurve]

One or more curves.

PrecisionRecallCurve

Bases: BaseModel

A single precision-recall curve.

Source code in pydantic_evals/pydantic_evals/reporting/analyses.py 
38
39
40
41
42
43
44
45
46
class PrecisionRecallCurve(BaseModel):
    """A single precision-recall curve."""

    name: str
    """Name of this curve (e.g., experiment name or evaluator name)."""
    points: list[PrecisionRecallPoint]
    """Points on the curve, ordered by threshold."""
    auc: float | None = None
    """Area under the precision-recall curve."""

name instance-attribute 

name: str

Name of this curve (e.g., experiment name or evaluator name).

points instance-attribute 

points: list[PrecisionRecallPoint]

Points on the curve, ordered by threshold.

auc class-attribute instance-attribute 

auc: float | None = None

Area under the precision-recall curve.

PrecisionRecallPoint

Bases: BaseModel

A single point on a precision-recall curve.

Source code in pydantic_evals/pydantic_evals/reporting/analyses.py 
30
31
32
33
34
35
class PrecisionRecallPoint(BaseModel):
    """A single point on a precision-recall curve."""

    threshold: float
    precision: float
    recall: float

ReportAnalysis module-attribute 

ReportAnalysis = Annotated[
    ConfusionMatrix
    | PrecisionRecall
    | ScalarResult
    | TableResult,
    Discriminator("type"),
]

Discriminated union of all report-level analysis types.

ScalarResult

Bases: BaseModel

A single scalar statistic (e.g., F1 score, accuracy, BLEU).

Source code in pydantic_evals/pydantic_evals/reporting/analyses.py 
59
60
61
62
63
64
65
66
67
class ScalarResult(BaseModel):
    """A single scalar statistic (e.g., F1 score, accuracy, BLEU)."""

    type: Literal['scalar'] = 'scalar'
    title: str
    description: str | None = None
    value: float | int
    unit: str | None = None
    """Optional unit label (e.g., '%', 'ms')."""

unit class-attribute instance-attribute 

unit: str | None = None

Optional unit label (e.g., '%', 'ms').

TableResult

Bases: BaseModel

A generic table of data (fallback for custom analyses).

Source code in pydantic_evals/pydantic_evals/reporting/analyses.py 
70
71
72
73
74
75
76
77
78
79
class TableResult(BaseModel):
    """A generic table of data (fallback for custom analyses)."""

    type: Literal['table'] = 'table'
    title: str
    description: str | None = None
    columns: list[str]
    """Column headers."""
    rows: list[list[str | int | float | bool | None]]
    """Row data, one list per row."""

columns instance-attribute 

columns: list[str]

Column headers.

rows instance-attribute 

rows: list[list[str | int | float | bool | None]]

Row data, one list per row.

ReportCase dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

A single case in an evaluation report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
@dataclass(kw_only=True)
class ReportCase(Generic[InputsT, OutputT, MetadataT]):
    """A single case in an evaluation report."""

    name: str
    """The name of the [case][pydantic_evals.Case]."""
    inputs: InputsT
    """The inputs to the task, from [`Case.inputs`][pydantic_evals.dataset.Case.inputs]."""
    metadata: MetadataT | None
    """Any metadata associated with the case, from [`Case.metadata`][pydantic_evals.dataset.Case.metadata]."""
    expected_output: OutputT | None
    """The expected output of the task, from [`Case.expected_output`][pydantic_evals.dataset.Case.expected_output]."""
    output: OutputT
    """The output of the task execution."""

    metrics: dict[str, float | int]
    attributes: dict[str, Any]

    scores: dict[str, EvaluationResult[int | float]]
    labels: dict[str, EvaluationResult[str]]
    assertions: dict[str, EvaluationResult[bool]]

    task_duration: float
    total_duration: float  # includes evaluator execution time

    source_case_name: str | None = None
    """The original case name before run-indexing. Serves as the aggregation key
    for multi-run experiments. None when repeat == 1."""

    trace_id: str | None = None
    """The trace ID of the case span."""
    span_id: str | None = None
    """The span ID of the case span."""

    evaluator_failures: list[EvaluatorFailure] = field(default_factory=list[EvaluatorFailure])

name instance-attribute 

name: str

The name of the case.

inputs instance-attribute 

inputs: InputsT

The inputs to the task, from Case.inputs.

metadata instance-attribute 

metadata: MetadataT | None

Any metadata associated with the case, from Case.metadata.

expected_output instance-attribute 

expected_output: OutputT | None

The expected output of the task, from Case.expected_output.

output instance-attribute 

output: OutputT

The output of the task execution.

source_case_name class-attribute instance-attribute 

source_case_name: str | None = None

The original case name before run-indexing. Serves as the aggregation key for multi-run experiments. None when repeat == 1.

trace_id class-attribute instance-attribute 

trace_id: str | None = None

The trace ID of the case span.

span_id class-attribute instance-attribute 

span_id: str | None = None

The span ID of the case span.

ReportCaseFailure dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

A single case in an evaluation report that failed due to an error during task execution.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
@dataclass(kw_only=True)
class ReportCaseFailure(Generic[InputsT, OutputT, MetadataT]):
    """A single case in an evaluation report that failed due to an error during task execution."""

    name: str
    """The name of the [case][pydantic_evals.Case]."""
    inputs: InputsT
    """The inputs to the task, from [`Case.inputs`][pydantic_evals.dataset.Case.inputs]."""
    metadata: MetadataT | None
    """Any metadata associated with the case, from [`Case.metadata`][pydantic_evals.dataset.Case.metadata]."""
    expected_output: OutputT | None
    """The expected output of the task, from [`Case.expected_output`][pydantic_evals.dataset.Case.expected_output]."""

    error_message: str
    """The message of the exception that caused the failure."""
    error_stacktrace: str
    """The stacktrace of the exception that caused the failure."""

    source_case_name: str | None = None
    """The original case name before run-indexing. Serves as the aggregation key
    for multi-run experiments. None when repeat == 1."""

    trace_id: str | None = None
    """The trace ID of the case span."""
    span_id: str | None = None
    """The span ID of the case span."""

name instance-attribute 

name: str

The name of the case.

inputs instance-attribute 

inputs: InputsT

The inputs to the task, from Case.inputs.

metadata instance-attribute 

metadata: MetadataT | None

Any metadata associated with the case, from Case.metadata.

expected_output instance-attribute 

expected_output: OutputT | None

The expected output of the task, from Case.expected_output.

error_message instance-attribute 

error_message: str

The message of the exception that caused the failure.

error_stacktrace instance-attribute 

error_stacktrace: str

The stacktrace of the exception that caused the failure.

source_case_name class-attribute instance-attribute 

source_case_name: str | None = None

The original case name before run-indexing. Serves as the aggregation key for multi-run experiments. None when repeat == 1.

trace_id class-attribute instance-attribute 

trace_id: str | None = None

The trace ID of the case span.

span_id class-attribute instance-attribute 

span_id: str | None = None

The span ID of the case span.

ReportCaseGroup dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

Grouped results from running the same case multiple times.

This is a computed view, not stored data. Obtain via EvaluationReport.case_groups().

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@dataclass(kw_only=True)
class ReportCaseGroup(Generic[InputsT, OutputT, MetadataT]):
    """Grouped results from running the same case multiple times.

    This is a computed view, not stored data. Obtain via
    `EvaluationReport.case_groups()`.
    """

    name: str
    """The original case name (shared across all runs)."""
    inputs: InputsT
    """The inputs (same for all runs)."""
    metadata: MetadataT | None
    """The metadata (same for all runs)."""
    expected_output: OutputT | None
    """The expected output (same for all runs)."""

    runs: Sequence[ReportCase[InputsT, OutputT, MetadataT]]
    """Individual run results."""
    failures: Sequence[ReportCaseFailure[InputsT, OutputT, MetadataT]]
    """Runs that failed with exceptions."""

    summary: ReportCaseAggregate
    """Aggregated statistics across runs."""

name instance-attribute 

name: str

The original case name (shared across all runs).

inputs instance-attribute 

inputs: InputsT

The inputs (same for all runs).

metadata instance-attribute 

metadata: MetadataT | None

The metadata (same for all runs).

expected_output instance-attribute 

expected_output: OutputT | None

The expected output (same for all runs).

runs instance-attribute 

runs: Sequence[ReportCase[InputsT, OutputT, MetadataT]]

Individual run results.

failures instance-attribute 

failures: Sequence[
    ReportCaseFailure[InputsT, OutputT, MetadataT]
]

Runs that failed with exceptions.

summary instance-attribute 

summary: ReportCaseAggregate

Aggregated statistics across runs.

ReportCaseAggregate

Bases: BaseModel

A synthetic case that summarizes a set of cases.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
class ReportCaseAggregate(BaseModel):
    """A synthetic case that summarizes a set of cases."""

    name: str

    scores: dict[str, float | int]
    labels: dict[str, dict[str, float]]
    metrics: dict[str, float | int]
    assertions: float | None
    task_duration: float
    total_duration: float

    @staticmethod
    def average(cases: list[ReportCase]) -> ReportCaseAggregate:
        """Produce a synthetic "summary" case by averaging quantitative attributes."""
        num_cases = len(cases)
        if num_cases == 0:
            return ReportCaseAggregate(
                name='Averages',
                scores={},
                labels={},
                metrics={},
                assertions=None,
                task_duration=0.0,
                total_duration=0.0,
            )

        def _scores_averages(scores_by_name: list[dict[str, int | float | bool]]) -> dict[str, float]:
            counts_by_name: dict[str, int] = defaultdict(int)
            sums_by_name: dict[str, float] = defaultdict(float)
            for sbn in scores_by_name:
                for name, score in sbn.items():
                    counts_by_name[name] += 1
                    sums_by_name[name] += score
            return {name: sums_by_name[name] / counts_by_name[name] for name in sums_by_name}

        def _labels_averages(labels_by_name: list[dict[str, str]]) -> dict[str, dict[str, float]]:
            counts_by_name: dict[str, int] = defaultdict(int)
            sums_by_name: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float))
            for lbn in labels_by_name:
                for name, label in lbn.items():
                    counts_by_name[name] += 1
                    sums_by_name[name][label] += 1
            return {
                name: {value: count / counts_by_name[name] for value, count in sums_by_name[name].items()}
                for name in sums_by_name
            }

        average_task_duration = sum(case.task_duration for case in cases) / num_cases
        average_total_duration = sum(case.total_duration for case in cases) / num_cases

        # average_assertions: dict[str, float] = _scores_averages([{k: v.value for k, v in case.scores.items()} for case in cases])
        average_scores: dict[str, float] = _scores_averages(
            [{k: v.value for k, v in case.scores.items()} for case in cases]
        )
        average_labels: dict[str, dict[str, float]] = _labels_averages(
            [{k: v.value for k, v in case.labels.items()} for case in cases]
        )
        average_metrics: dict[str, float] = _scores_averages([case.metrics for case in cases])

        average_assertions: float | None = None
        n_assertions = sum(len(case.assertions) for case in cases)
        if n_assertions > 0:
            n_passing = sum(1 for case in cases for assertion in case.assertions.values() if assertion.value)
            average_assertions = n_passing / n_assertions

        return ReportCaseAggregate(
            name='Averages',
            scores=average_scores,
            labels=average_labels,
            metrics=average_metrics,
            assertions=average_assertions,
            task_duration=average_task_duration,
            total_duration=average_total_duration,
        )

    @staticmethod
    def average_from_aggregates(aggregates: list[ReportCaseAggregate]) -> ReportCaseAggregate:
        """Average across multiple aggregates (used for multi-run experiment summaries)."""
        if not aggregates:
            return ReportCaseAggregate(
                name='Averages',
                scores={},
                labels={},
                metrics={},
                assertions=None,
                task_duration=0.0,
                total_duration=0.0,
            )

        def _avg_numeric_dicts(dicts: list[dict[str, float | int]]) -> dict[str, float | int]:
            all_keys: set[str] = set()
            for d in dicts:
                all_keys.update(d)
            return {key: sum(vals) / len(vals) for key in all_keys if (vals := [d[key] for d in dicts if key in d])}

        avg_scores = _avg_numeric_dicts([a.scores for a in aggregates])
        avg_metrics = _avg_numeric_dicts([a.metrics for a in aggregates])

        # Average labels (average the distribution dicts)
        all_label_keys: set[str] = set()
        for a in aggregates:
            all_label_keys.update(a.labels)
        avg_labels: dict[str, dict[str, float]] = {}
        for key in all_label_keys:
            combined: dict[str, float] = {}
            count = 0
            for a in aggregates:
                if key in a.labels:
                    count += 1
                    for label_val, freq in a.labels[key].items():
                        combined[label_val] = combined.get(label_val, 0.0) + freq
            avg_labels[key] = {k: v / count for k, v in combined.items()}

        # Average assertions
        assertion_values = [a.assertions for a in aggregates if a.assertions is not None]
        avg_assertions: float | None = None
        if assertion_values:
            avg_assertions = sum(assertion_values) / len(assertion_values)

        # Average durations
        task_durs = [a.task_duration for a in aggregates]
        total_durs = [a.total_duration for a in aggregates]

        return ReportCaseAggregate(
            name='Averages',
            scores=avg_scores,
            labels=avg_labels,
            metrics=avg_metrics,
            assertions=avg_assertions,
            task_duration=sum(task_durs) / len(task_durs),
            total_duration=sum(total_durs) / len(total_durs),
        )

average staticmethod 

average(cases: list[ReportCase]) -> ReportCaseAggregate

Produce a synthetic "summary" case by averaging quantitative attributes.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
@staticmethod
def average(cases: list[ReportCase]) -> ReportCaseAggregate:
    """Produce a synthetic "summary" case by averaging quantitative attributes."""
    num_cases = len(cases)
    if num_cases == 0:
        return ReportCaseAggregate(
            name='Averages',
            scores={},
            labels={},
            metrics={},
            assertions=None,
            task_duration=0.0,
            total_duration=0.0,
        )

    def _scores_averages(scores_by_name: list[dict[str, int | float | bool]]) -> dict[str, float]:
        counts_by_name: dict[str, int] = defaultdict(int)
        sums_by_name: dict[str, float] = defaultdict(float)
        for sbn in scores_by_name:
            for name, score in sbn.items():
                counts_by_name[name] += 1
                sums_by_name[name] += score
        return {name: sums_by_name[name] / counts_by_name[name] for name in sums_by_name}

    def _labels_averages(labels_by_name: list[dict[str, str]]) -> dict[str, dict[str, float]]:
        counts_by_name: dict[str, int] = defaultdict(int)
        sums_by_name: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float))
        for lbn in labels_by_name:
            for name, label in lbn.items():
                counts_by_name[name] += 1
                sums_by_name[name][label] += 1
        return {
            name: {value: count / counts_by_name[name] for value, count in sums_by_name[name].items()}
            for name in sums_by_name
        }

    average_task_duration = sum(case.task_duration for case in cases) / num_cases
    average_total_duration = sum(case.total_duration for case in cases) / num_cases

    # average_assertions: dict[str, float] = _scores_averages([{k: v.value for k, v in case.scores.items()} for case in cases])
    average_scores: dict[str, float] = _scores_averages(
        [{k: v.value for k, v in case.scores.items()} for case in cases]
    )
    average_labels: dict[str, dict[str, float]] = _labels_averages(
        [{k: v.value for k, v in case.labels.items()} for case in cases]
    )
    average_metrics: dict[str, float] = _scores_averages([case.metrics for case in cases])

    average_assertions: float | None = None
    n_assertions = sum(len(case.assertions) for case in cases)
    if n_assertions > 0:
        n_passing = sum(1 for case in cases for assertion in case.assertions.values() if assertion.value)
        average_assertions = n_passing / n_assertions

    return ReportCaseAggregate(
        name='Averages',
        scores=average_scores,
        labels=average_labels,
        metrics=average_metrics,
        assertions=average_assertions,
        task_duration=average_task_duration,
        total_duration=average_total_duration,
    )

average_from_aggregates staticmethod 

average_from_aggregates(
    aggregates: list[ReportCaseAggregate],
) -> ReportCaseAggregate

Average across multiple aggregates (used for multi-run experiment summaries).

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
@staticmethod
def average_from_aggregates(aggregates: list[ReportCaseAggregate]) -> ReportCaseAggregate:
    """Average across multiple aggregates (used for multi-run experiment summaries)."""
    if not aggregates:
        return ReportCaseAggregate(
            name='Averages',
            scores={},
            labels={},
            metrics={},
            assertions=None,
            task_duration=0.0,
            total_duration=0.0,
        )

    def _avg_numeric_dicts(dicts: list[dict[str, float | int]]) -> dict[str, float | int]:
        all_keys: set[str] = set()
        for d in dicts:
            all_keys.update(d)
        return {key: sum(vals) / len(vals) for key in all_keys if (vals := [d[key] for d in dicts if key in d])}

    avg_scores = _avg_numeric_dicts([a.scores for a in aggregates])
    avg_metrics = _avg_numeric_dicts([a.metrics for a in aggregates])

    # Average labels (average the distribution dicts)
    all_label_keys: set[str] = set()
    for a in aggregates:
        all_label_keys.update(a.labels)
    avg_labels: dict[str, dict[str, float]] = {}
    for key in all_label_keys:
        combined: dict[str, float] = {}
        count = 0
        for a in aggregates:
            if key in a.labels:
                count += 1
                for label_val, freq in a.labels[key].items():
                    combined[label_val] = combined.get(label_val, 0.0) + freq
        avg_labels[key] = {k: v / count for k, v in combined.items()}

    # Average assertions
    assertion_values = [a.assertions for a in aggregates if a.assertions is not None]
    avg_assertions: float | None = None
    if assertion_values:
        avg_assertions = sum(assertion_values) / len(assertion_values)

    # Average durations
    task_durs = [a.task_duration for a in aggregates]
    total_durs = [a.total_duration for a in aggregates]

    return ReportCaseAggregate(
        name='Averages',
        scores=avg_scores,
        labels=avg_labels,
        metrics=avg_metrics,
        assertions=avg_assertions,
        task_duration=sum(task_durs) / len(task_durs),
        total_duration=sum(total_durs) / len(total_durs),
    )

EvaluationReport dataclass

Bases: Generic[InputsT, OutputT, MetadataT]

A report of the results of evaluating a model on a set of cases.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
@dataclass(kw_only=True)
class EvaluationReport(Generic[InputsT, OutputT, MetadataT]):
    """A report of the results of evaluating a model on a set of cases."""

    name: str
    """The name of the report."""

    cases: list[ReportCase[InputsT, OutputT, MetadataT]]
    """The cases in the report."""
    failures: list[ReportCaseFailure[InputsT, OutputT, MetadataT]] = field(
        default_factory=list[ReportCaseFailure[InputsT, OutputT, MetadataT]]
    )
    """The failures in the report. These are cases where task execution raised an exception."""

    analyses: list[ReportAnalysis] = field(default_factory=list[ReportAnalysis])
    """Experiment-wide analyses produced by report evaluators."""

    report_evaluator_failures: list[EvaluatorFailure] = field(default_factory=list[EvaluatorFailure])
    """Failures from report evaluators that raised exceptions."""

    experiment_metadata: dict[str, Any] | None = None
    """Metadata associated with the specific experiment represented by this report."""
    trace_id: str | None = None
    """The trace ID of the evaluation."""
    span_id: str | None = None
    """The span ID of the evaluation."""

    def case_groups(self) -> list[ReportCaseGroup[InputsT, OutputT, MetadataT]] | None:
        """Group cases by source_case_name and compute per-group aggregates.

        Returns None if no cases have source_case_name set (i.e., single-run experiment).
        """
        if not any(c.source_case_name for c in self.cases) and not any(f.source_case_name for f in self.failures):
            return None

        groups: dict[
            str,
            tuple[list[ReportCase[InputsT, OutputT, MetadataT]], list[ReportCaseFailure[InputsT, OutputT, MetadataT]]],
        ] = {}
        for case in self.cases:
            key = case.source_case_name or case.name
            groups.setdefault(key, ([], []))[0].append(case)
        for failure in self.failures:
            key = failure.source_case_name or failure.name
            groups.setdefault(key, ([], []))[1].append(failure)

        result: list[ReportCaseGroup[InputsT, OutputT, MetadataT]] = []
        for group_name, (runs, failures) in groups.items():
            first: ReportCase[InputsT, OutputT, MetadataT] | ReportCaseFailure[InputsT, OutputT, MetadataT] = (
                runs[0] if runs else failures[0]
            )
            result.append(
                ReportCaseGroup(
                    name=group_name,
                    inputs=first.inputs,
                    metadata=first.metadata,
                    expected_output=first.expected_output,
                    runs=runs,
                    failures=failures,
                    summary=ReportCaseAggregate.average(list(runs)),
                )
            )
        return result

    def averages(self) -> ReportCaseAggregate | None:
        groups = self.case_groups()
        if groups is not None:
            non_empty_summaries = [g.summary for g in groups if g.runs]
            return ReportCaseAggregate.average_from_aggregates(non_empty_summaries) if non_empty_summaries else None
        elif self.cases:
            return ReportCaseAggregate.average(self.cases)
        return None

    def render(
        self,
        width: int | None = None,
        baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None,
        *,
        include_input: bool = False,
        include_metadata: bool = False,
        include_expected_output: bool = False,
        include_output: bool = False,
        include_durations: bool = True,
        include_total_duration: bool = False,
        include_removed_cases: bool = False,
        include_averages: bool = True,
        include_errors: bool = True,
        include_error_stacktrace: bool = False,
        include_evaluator_failures: bool = True,
        include_analyses: bool = True,
        input_config: RenderValueConfig | None = None,
        metadata_config: RenderValueConfig | None = None,
        output_config: RenderValueConfig | None = None,
        score_configs: dict[str, RenderNumberConfig] | None = None,
        label_configs: dict[str, RenderValueConfig] | None = None,
        metric_configs: dict[str, RenderNumberConfig] | None = None,
        duration_config: RenderNumberConfig | None = None,
        include_reasons: bool = False,
    ) -> str:
        """Render this report to a nicely-formatted string, optionally comparing it to a baseline report.

        If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.
        """
        io_file = StringIO()
        console = Console(width=width, file=io_file)
        self.print(
            width=width,
            baseline=baseline,
            console=console,
            include_input=include_input,
            include_metadata=include_metadata,
            include_expected_output=include_expected_output,
            include_output=include_output,
            include_durations=include_durations,
            include_total_duration=include_total_duration,
            include_removed_cases=include_removed_cases,
            include_averages=include_averages,
            include_errors=include_errors,
            include_error_stacktrace=include_error_stacktrace,
            include_evaluator_failures=include_evaluator_failures,
            include_analyses=include_analyses,
            input_config=input_config,
            metadata_config=metadata_config,
            output_config=output_config,
            score_configs=score_configs,
            label_configs=label_configs,
            metric_configs=metric_configs,
            duration_config=duration_config,
            include_reasons=include_reasons,
        )
        return io_file.getvalue()

    def print(
        self,
        width: int | None = None,
        baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None,
        *,
        console: Console | None = None,
        include_input: bool = False,
        include_metadata: bool = False,
        include_expected_output: bool = False,
        include_output: bool = False,
        include_durations: bool = True,
        include_total_duration: bool = False,
        include_removed_cases: bool = False,
        include_averages: bool = True,
        include_errors: bool = True,
        include_error_stacktrace: bool = False,
        include_evaluator_failures: bool = True,
        include_analyses: bool = True,
        input_config: RenderValueConfig | None = None,
        metadata_config: RenderValueConfig | None = None,
        output_config: RenderValueConfig | None = None,
        score_configs: dict[str, RenderNumberConfig] | None = None,
        label_configs: dict[str, RenderValueConfig] | None = None,
        metric_configs: dict[str, RenderNumberConfig] | None = None,
        duration_config: RenderNumberConfig | None = None,
        include_reasons: bool = False,
    ) -> None:
        """Print this report to the console, optionally comparing it to a baseline report.

        If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.
        """
        if console is None:  # pragma: no branch
            console = Console(width=width)

        metadata_panel = self._metadata_panel(baseline=baseline)
        renderable: RenderableType = self.console_table(
            baseline=baseline,
            include_input=include_input,
            include_metadata=include_metadata,
            include_expected_output=include_expected_output,
            include_output=include_output,
            include_durations=include_durations,
            include_total_duration=include_total_duration,
            include_removed_cases=include_removed_cases,
            include_averages=include_averages,
            include_evaluator_failures=include_evaluator_failures,
            input_config=input_config,
            metadata_config=metadata_config,
            output_config=output_config,
            score_configs=score_configs,
            label_configs=label_configs,
            metric_configs=metric_configs,
            duration_config=duration_config,
            include_reasons=include_reasons,
            with_title=not metadata_panel,
        )
        # Wrap table with experiment metadata panel if present
        if metadata_panel:
            renderable = Group(metadata_panel, renderable)
        console.print(renderable)
        if include_analyses and self.analyses:
            for analysis in self.analyses:
                console.print(_render_analysis(analysis))
        if include_evaluator_failures and self.report_evaluator_failures:
            console.print(
                Text('\nReport Evaluator Failures:', style='bold red'),
            )
            for failure in self.report_evaluator_failures:
                msg = f'  {failure.name}: {failure.error_message}'
                console.print(Text(msg, style='red'))
        if include_errors and self.failures:  # pragma: no cover
            failures_table = self.failures_table(
                include_input=include_input,
                include_metadata=include_metadata,
                include_expected_output=include_expected_output,
                include_error_message=True,
                include_error_stacktrace=include_error_stacktrace,
                input_config=input_config,
                metadata_config=metadata_config,
            )
            console.print(failures_table, style='red')

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def console_table(
        self,
        baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None,
        *,
        include_input: bool = False,
        include_metadata: bool = False,
        include_expected_output: bool = False,
        include_output: bool = False,
        include_durations: bool = True,
        include_total_duration: bool = False,
        include_removed_cases: bool = False,
        include_averages: bool = True,
        include_evaluator_failures: bool = True,
        input_config: RenderValueConfig | None = None,
        metadata_config: RenderValueConfig | None = None,
        output_config: RenderValueConfig | None = None,
        score_configs: dict[str, RenderNumberConfig] | None = None,
        label_configs: dict[str, RenderValueConfig] | None = None,
        metric_configs: dict[str, RenderNumberConfig] | None = None,
        duration_config: RenderNumberConfig | None = None,
        include_reasons: bool = False,
        with_title: bool = True,
    ) -> Table:
        """Return a table containing the data from this report.

        If a baseline is provided, returns a diff between this report and the baseline report.
        Optionally include input and output details.
        """
        renderer = EvaluationRenderer(
            include_input=include_input,
            include_metadata=include_metadata,
            include_expected_output=include_expected_output,
            include_output=include_output,
            include_durations=include_durations,
            include_total_duration=include_total_duration,
            include_removed_cases=include_removed_cases,
            include_averages=include_averages,
            include_error_message=False,
            include_error_stacktrace=False,
            include_evaluator_failures=include_evaluator_failures,
            input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})},
            metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})},
            output_config=output_config or _DEFAULT_VALUE_CONFIG,
            score_configs=score_configs or {},
            label_configs=label_configs or {},
            metric_configs=metric_configs or {},
            duration_config=duration_config or _DEFAULT_DURATION_CONFIG,
            include_reasons=include_reasons,
        )
        if baseline is None:
            return renderer.build_table(self, with_title=with_title)
        else:
            return renderer.build_diff_table(self, baseline, with_title=with_title)

    def _metadata_panel(
        self, baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None
    ) -> RenderableType | None:
        """Wrap a table with an experiment metadata panel if metadata exists.

        Args:
            table: The table to wrap
            baseline: Optional baseline report for diff metadata

        Returns:
            Either the table unchanged or a Group with Panel and Table
        """
        if baseline is None:
            # Single report - show metadata if present
            if self.experiment_metadata:
                metadata_text = Text()
                items = list(self.experiment_metadata.items())
                for i, (key, value) in enumerate(items):
                    metadata_text.append(f'{key}: {value}', style='dim')
                    if i < len(items) - 1:
                        metadata_text.append('\n')
                return Panel(
                    metadata_text,
                    title=f'Evaluation Summary: {self.name}',
                    title_align='left',
                    border_style='dim',
                    padding=(0, 1),
                    expand=False,
                )
        else:
            # Diff report - show metadata diff if either has metadata
            if self.experiment_metadata or baseline.experiment_metadata:
                diff_name = baseline.name if baseline.name == self.name else f'{baseline.name} → {self.name}'
                metadata_text = Text()
                lines_styles: list[tuple[str, str]] = []
                if baseline.experiment_metadata and self.experiment_metadata:
                    # Collect all keys from both
                    all_keys = sorted(set(baseline.experiment_metadata.keys()) | set(self.experiment_metadata.keys()))
                    for key in all_keys:
                        baseline_val = baseline.experiment_metadata.get(key)
                        report_val = self.experiment_metadata.get(key)
                        if baseline_val == report_val:
                            lines_styles.append((f'{key}: {report_val}', 'dim'))
                        elif baseline_val is None:
                            lines_styles.append((f'+ {key}: {report_val}', 'green'))
                        elif report_val is None:
                            lines_styles.append((f'- {key}: {baseline_val}', 'red'))
                        else:
                            lines_styles.append((f'{key}: {baseline_val} → {report_val}', 'yellow'))
                elif self.experiment_metadata:
                    lines_styles = [(f'+ {k}: {v}', 'green') for k, v in self.experiment_metadata.items()]
                else:  # baseline.experiment_metadata only
                    assert baseline.experiment_metadata is not None
                    lines_styles = [(f'- {k}: {v}', 'red') for k, v in baseline.experiment_metadata.items()]

                for i, (line, style) in enumerate(lines_styles):
                    metadata_text.append(line, style=style)
                    if i < len(lines_styles) - 1:
                        metadata_text.append('\n')

                return Panel(
                    metadata_text,
                    title=f'Evaluation Diff: {diff_name}',
                    title_align='left',
                    border_style='dim',
                    padding=(0, 1),
                    expand=False,
                )

        return None

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def failures_table(
        self,
        *,
        include_input: bool = False,
        include_metadata: bool = False,
        include_expected_output: bool = False,
        include_error_message: bool = True,
        include_error_stacktrace: bool = True,
        input_config: RenderValueConfig | None = None,
        metadata_config: RenderValueConfig | None = None,
    ) -> Table:
        """Return a table containing the failures in this report."""
        renderer = EvaluationRenderer(
            include_input=include_input,
            include_metadata=include_metadata,
            include_expected_output=include_expected_output,
            include_output=False,
            include_durations=False,
            include_total_duration=False,
            include_removed_cases=False,
            include_averages=False,
            input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})},
            metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})},
            output_config=_DEFAULT_VALUE_CONFIG,
            score_configs={},
            label_configs={},
            metric_configs={},
            duration_config=_DEFAULT_DURATION_CONFIG,
            include_reasons=False,
            include_error_message=include_error_message,
            include_error_stacktrace=include_error_stacktrace,
            include_evaluator_failures=False,  # Not applicable for failures table
        )
        return renderer.build_failures_table(self)

    def __str__(self) -> str:  # pragma: lax no cover
        """Return a string representation of the report."""
        return self.render()

name instance-attribute 

name: str

The name of the report.

cases instance-attribute 

cases: list[ReportCase[InputsT, OutputT, MetadataT]]

The cases in the report.

failures class-attribute instance-attribute 

failures: list[
    ReportCaseFailure[InputsT, OutputT, MetadataT]
] = field(
    default_factory=list[
        ReportCaseFailure[InputsT, OutputT, MetadataT]
    ]
)

The failures in the report. These are cases where task execution raised an exception.

analyses class-attribute instance-attribute 

analyses: list[ReportAnalysis] = field(
    default_factory=list[ReportAnalysis]
)

Experiment-wide analyses produced by report evaluators.

report_evaluator_failures class-attribute instance-attribute 

report_evaluator_failures: list[EvaluatorFailure] = field(
    default_factory=list[EvaluatorFailure]
)

Failures from report evaluators that raised exceptions.

experiment_metadata class-attribute instance-attribute 

experiment_metadata: dict[str, Any] | None = None

Metadata associated with the specific experiment represented by this report.

trace_id class-attribute instance-attribute 

trace_id: str | None = None

The trace ID of the evaluation.

span_id class-attribute instance-attribute 

span_id: str | None = None

The span ID of the evaluation.

case_groups 

case_groups() -> (
    list[ReportCaseGroup[InputsT, OutputT, MetadataT]]
    | None
)

Group cases by source_case_name and compute per-group aggregates.

Returns None if no cases have source_case_name set (i.e., single-run experiment).

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
def case_groups(self) -> list[ReportCaseGroup[InputsT, OutputT, MetadataT]] | None:
    """Group cases by source_case_name and compute per-group aggregates.

    Returns None if no cases have source_case_name set (i.e., single-run experiment).
    """
    if not any(c.source_case_name for c in self.cases) and not any(f.source_case_name for f in self.failures):
        return None

    groups: dict[
        str,
        tuple[list[ReportCase[InputsT, OutputT, MetadataT]], list[ReportCaseFailure[InputsT, OutputT, MetadataT]]],
    ] = {}
    for case in self.cases:
        key = case.source_case_name or case.name
        groups.setdefault(key, ([], []))[0].append(case)
    for failure in self.failures:
        key = failure.source_case_name or failure.name
        groups.setdefault(key, ([], []))[1].append(failure)

    result: list[ReportCaseGroup[InputsT, OutputT, MetadataT]] = []
    for group_name, (runs, failures) in groups.items():
        first: ReportCase[InputsT, OutputT, MetadataT] | ReportCaseFailure[InputsT, OutputT, MetadataT] = (
            runs[0] if runs else failures[0]
        )
        result.append(
            ReportCaseGroup(
                name=group_name,
                inputs=first.inputs,
                metadata=first.metadata,
                expected_output=first.expected_output,
                runs=runs,
                failures=failures,
                summary=ReportCaseAggregate.average(list(runs)),
            )
        )
    return result

render 

render(
    width: int | None = None,
    baseline: (
        EvaluationReport[InputsT, OutputT, MetadataT] | None
    ) = None,
    *,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_output: bool = False,
    include_durations: bool = True,
    include_total_duration: bool = False,
    include_removed_cases: bool = False,
    include_averages: bool = True,
    include_errors: bool = True,
    include_error_stacktrace: bool = False,
    include_evaluator_failures: bool = True,
    include_analyses: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
    output_config: RenderValueConfig | None = None,
    score_configs: (
        dict[str, RenderNumberConfig] | None
    ) = None,
    label_configs: (
        dict[str, RenderValueConfig] | None
    ) = None,
    metric_configs: (
        dict[str, RenderNumberConfig] | None
    ) = None,
    duration_config: RenderNumberConfig | None = None,
    include_reasons: bool = False
) -> str

Render this report to a nicely-formatted string, optionally comparing it to a baseline report.

If you want more control over the output, use console_table instead and pass it to rich.Console.print.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
def render(
    self,
    width: int | None = None,
    baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None,
    *,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_output: bool = False,
    include_durations: bool = True,
    include_total_duration: bool = False,
    include_removed_cases: bool = False,
    include_averages: bool = True,
    include_errors: bool = True,
    include_error_stacktrace: bool = False,
    include_evaluator_failures: bool = True,
    include_analyses: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
    output_config: RenderValueConfig | None = None,
    score_configs: dict[str, RenderNumberConfig] | None = None,
    label_configs: dict[str, RenderValueConfig] | None = None,
    metric_configs: dict[str, RenderNumberConfig] | None = None,
    duration_config: RenderNumberConfig | None = None,
    include_reasons: bool = False,
) -> str:
    """Render this report to a nicely-formatted string, optionally comparing it to a baseline report.

    If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.
    """
    io_file = StringIO()
    console = Console(width=width, file=io_file)
    self.print(
        width=width,
        baseline=baseline,
        console=console,
        include_input=include_input,
        include_metadata=include_metadata,
        include_expected_output=include_expected_output,
        include_output=include_output,
        include_durations=include_durations,
        include_total_duration=include_total_duration,
        include_removed_cases=include_removed_cases,
        include_averages=include_averages,
        include_errors=include_errors,
        include_error_stacktrace=include_error_stacktrace,
        include_evaluator_failures=include_evaluator_failures,
        include_analyses=include_analyses,
        input_config=input_config,
        metadata_config=metadata_config,
        output_config=output_config,
        score_configs=score_configs,
        label_configs=label_configs,
        metric_configs=metric_configs,
        duration_config=duration_config,
        include_reasons=include_reasons,
    )
    return io_file.getvalue()

print 

print(
    width: int | None = None,
    baseline: (
        EvaluationReport[InputsT, OutputT, MetadataT] | None
    ) = None,
    *,
    console: Console | None = None,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_output: bool = False,
    include_durations: bool = True,
    include_total_duration: bool = False,
    include_removed_cases: bool = False,
    include_averages: bool = True,
    include_errors: bool = True,
    include_error_stacktrace: bool = False,
    include_evaluator_failures: bool = True,
    include_analyses: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
    output_config: RenderValueConfig | None = None,
    score_configs: (
        dict[str, RenderNumberConfig] | None
    ) = None,
    label_configs: (
        dict[str, RenderValueConfig] | None
    ) = None,
    metric_configs: (
        dict[str, RenderNumberConfig] | None
    ) = None,
    duration_config: RenderNumberConfig | None = None,
    include_reasons: bool = False
) -> None

Print this report to the console, optionally comparing it to a baseline report.

If you want more control over the output, use console_table instead and pass it to rich.Console.print.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
def print(
    self,
    width: int | None = None,
    baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None,
    *,
    console: Console | None = None,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_output: bool = False,
    include_durations: bool = True,
    include_total_duration: bool = False,
    include_removed_cases: bool = False,
    include_averages: bool = True,
    include_errors: bool = True,
    include_error_stacktrace: bool = False,
    include_evaluator_failures: bool = True,
    include_analyses: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
    output_config: RenderValueConfig | None = None,
    score_configs: dict[str, RenderNumberConfig] | None = None,
    label_configs: dict[str, RenderValueConfig] | None = None,
    metric_configs: dict[str, RenderNumberConfig] | None = None,
    duration_config: RenderNumberConfig | None = None,
    include_reasons: bool = False,
) -> None:
    """Print this report to the console, optionally comparing it to a baseline report.

    If you want more control over the output, use `console_table` instead and pass it to `rich.Console.print`.
    """
    if console is None:  # pragma: no branch
        console = Console(width=width)

    metadata_panel = self._metadata_panel(baseline=baseline)
    renderable: RenderableType = self.console_table(
        baseline=baseline,
        include_input=include_input,
        include_metadata=include_metadata,
        include_expected_output=include_expected_output,
        include_output=include_output,
        include_durations=include_durations,
        include_total_duration=include_total_duration,
        include_removed_cases=include_removed_cases,
        include_averages=include_averages,
        include_evaluator_failures=include_evaluator_failures,
        input_config=input_config,
        metadata_config=metadata_config,
        output_config=output_config,
        score_configs=score_configs,
        label_configs=label_configs,
        metric_configs=metric_configs,
        duration_config=duration_config,
        include_reasons=include_reasons,
        with_title=not metadata_panel,
    )
    # Wrap table with experiment metadata panel if present
    if metadata_panel:
        renderable = Group(metadata_panel, renderable)
    console.print(renderable)
    if include_analyses and self.analyses:
        for analysis in self.analyses:
            console.print(_render_analysis(analysis))
    if include_evaluator_failures and self.report_evaluator_failures:
        console.print(
            Text('\nReport Evaluator Failures:', style='bold red'),
        )
        for failure in self.report_evaluator_failures:
            msg = f'  {failure.name}: {failure.error_message}'
            console.print(Text(msg, style='red'))
    if include_errors and self.failures:  # pragma: no cover
        failures_table = self.failures_table(
            include_input=include_input,
            include_metadata=include_metadata,
            include_expected_output=include_expected_output,
            include_error_message=True,
            include_error_stacktrace=include_error_stacktrace,
            input_config=input_config,
            metadata_config=metadata_config,
        )
        console.print(failures_table, style='red')

console_table 

console_table(
    baseline: (
        EvaluationReport[InputsT, OutputT, MetadataT] | None
    ) = None,
    *,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_output: bool = False,
    include_durations: bool = True,
    include_total_duration: bool = False,
    include_removed_cases: bool = False,
    include_averages: bool = True,
    include_evaluator_failures: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
    output_config: RenderValueConfig | None = None,
    score_configs: (
        dict[str, RenderNumberConfig] | None
    ) = None,
    label_configs: (
        dict[str, RenderValueConfig] | None
    ) = None,
    metric_configs: (
        dict[str, RenderNumberConfig] | None
    ) = None,
    duration_config: RenderNumberConfig | None = None,
    include_reasons: bool = False,
    with_title: bool = True
) -> Table

Return a table containing the data from this report.

If a baseline is provided, returns a diff between this report and the baseline report. Optionally include input and output details.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
def console_table(
    self,
    baseline: EvaluationReport[InputsT, OutputT, MetadataT] | None = None,
    *,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_output: bool = False,
    include_durations: bool = True,
    include_total_duration: bool = False,
    include_removed_cases: bool = False,
    include_averages: bool = True,
    include_evaluator_failures: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
    output_config: RenderValueConfig | None = None,
    score_configs: dict[str, RenderNumberConfig] | None = None,
    label_configs: dict[str, RenderValueConfig] | None = None,
    metric_configs: dict[str, RenderNumberConfig] | None = None,
    duration_config: RenderNumberConfig | None = None,
    include_reasons: bool = False,
    with_title: bool = True,
) -> Table:
    """Return a table containing the data from this report.

    If a baseline is provided, returns a diff between this report and the baseline report.
    Optionally include input and output details.
    """
    renderer = EvaluationRenderer(
        include_input=include_input,
        include_metadata=include_metadata,
        include_expected_output=include_expected_output,
        include_output=include_output,
        include_durations=include_durations,
        include_total_duration=include_total_duration,
        include_removed_cases=include_removed_cases,
        include_averages=include_averages,
        include_error_message=False,
        include_error_stacktrace=False,
        include_evaluator_failures=include_evaluator_failures,
        input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})},
        metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})},
        output_config=output_config or _DEFAULT_VALUE_CONFIG,
        score_configs=score_configs or {},
        label_configs=label_configs or {},
        metric_configs=metric_configs or {},
        duration_config=duration_config or _DEFAULT_DURATION_CONFIG,
        include_reasons=include_reasons,
    )
    if baseline is None:
        return renderer.build_table(self, with_title=with_title)
    else:
        return renderer.build_diff_table(self, baseline, with_title=with_title)

failures_table 

failures_table(
    *,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_error_message: bool = True,
    include_error_stacktrace: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None
) -> Table

Return a table containing the failures in this report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
def failures_table(
    self,
    *,
    include_input: bool = False,
    include_metadata: bool = False,
    include_expected_output: bool = False,
    include_error_message: bool = True,
    include_error_stacktrace: bool = True,
    input_config: RenderValueConfig | None = None,
    metadata_config: RenderValueConfig | None = None,
) -> Table:
    """Return a table containing the failures in this report."""
    renderer = EvaluationRenderer(
        include_input=include_input,
        include_metadata=include_metadata,
        include_expected_output=include_expected_output,
        include_output=False,
        include_durations=False,
        include_total_duration=False,
        include_removed_cases=False,
        include_averages=False,
        input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})},
        metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})},
        output_config=_DEFAULT_VALUE_CONFIG,
        score_configs={},
        label_configs={},
        metric_configs={},
        duration_config=_DEFAULT_DURATION_CONFIG,
        include_reasons=False,
        include_error_message=include_error_message,
        include_error_stacktrace=include_error_stacktrace,
        include_evaluator_failures=False,  # Not applicable for failures table
    )
    return renderer.build_failures_table(self)

__str__ 

__str__() -> str

Return a string representation of the report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
674
675
676
def __str__(self) -> str:  # pragma: lax no cover
    """Return a string representation of the report."""
    return self.render()

RenderValueConfig

Bases: TypedDict

A configuration for rendering a values in an Evaluation report.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
682
683
684
685
686
687
688
class RenderValueConfig(TypedDict, total=False):
    """A configuration for rendering a values in an Evaluation report."""

    value_formatter: str | Callable[[Any], str]
    diff_checker: Callable[[Any, Any], bool] | None
    diff_formatter: Callable[[Any, Any], str | None] | None
    diff_style: str

RenderNumberConfig

Bases: TypedDict

A configuration for rendering a particular score or metric in an Evaluation report.

See the implementation of _RenderNumber for more clarity on how these parameters affect the rendering.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
class RenderNumberConfig(TypedDict, total=False):
    """A configuration for rendering a particular score or metric in an Evaluation report.

    See the implementation of `_RenderNumber` for more clarity on how these parameters affect the rendering.
    """

    value_formatter: str | Callable[[float | int], str]
    """The logic to use for formatting values.

    * If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures.
    * You can also use a custom string format spec, e.g. '{:.3f}'
    * You can also use a custom function, e.g. lambda x: f'{x:.3f}'
    """
    diff_formatter: str | Callable[[float | int, float | int], str | None] | None
    """The logic to use for formatting details about the diff.

    The strings produced by the value_formatter will always be included in the reports, but the diff_formatter is
    used to produce additional text about the difference between the old and new values, such as the absolute or
    relative difference.

    * If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four
        significant figures, and will include the percentage change.
    * You can also use a custom string format spec, e.g. '{:+.3f}'
    * You can also use a custom function, e.g. lambda x: f'{x:+.3f}'.
        If this function returns None, no extra diff text will be added.
    * You can also use None to never generate extra diff text.
    """
    diff_atol: float
    """The absolute tolerance for considering a difference "significant".

    A difference is "significant" if `abs(new - old) < self.diff_atol + self.diff_rtol * abs(old)`.

    If a difference is not significant, it will not have the diff styles applied. Note that we still show
    both the rendered before and after values in the diff any time they differ, even if the difference is not
    significant. (If the rendered values are exactly the same, we only show the value once.)

    If not provided, use 1e-6.
    """
    diff_rtol: float
    """The relative tolerance for considering a difference "significant".

    See the description of `diff_atol` for more details about what makes a difference "significant".

    If not provided, use 0.001 if all values are ints, otherwise 0.05.
    """
    diff_increase_style: str
    """The style to apply to diffed values that have a significant increase.

    See the description of `diff_atol` for more details about what makes a difference "significant".

    If not provided, use green for scores and red for metrics. You can also use arbitrary `rich` styles, such as "bold red".
    """
    diff_decrease_style: str
    """The style to apply to diffed values that have significant decrease.

    See the description of `diff_atol` for more details about what makes a difference "significant".

    If not provided, use red for scores and green for metrics. You can also use arbitrary `rich` styles, such as "bold red".
    """

value_formatter instance-attribute 

value_formatter: str | Callable[[float | int], str]

The logic to use for formatting values.

  • If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures.
  • You can also use a custom string format spec, e.g. '{:.3f}'
  • You can also use a custom function, e.g. lambda x: f'{x:.3f}'

diff_formatter instance-attribute 

diff_formatter: (
    str
    | Callable[[float | int, float | int], str | None]
    | None
)

The logic to use for formatting details about the diff.

The strings produced by the value_formatter will always be included in the reports, but the diff_formatter is used to produce additional text about the difference between the old and new values, such as the absolute or relative difference.

  • If not provided, format as ints if all values are ints, otherwise at least one decimal place and at least four significant figures, and will include the percentage change.
  • You can also use a custom string format spec, e.g. '{:+.3f}'
  • You can also use a custom function, e.g. lambda x: f'{x:+.3f}'. If this function returns None, no extra diff text will be added.
  • You can also use None to never generate extra diff text.

diff_atol instance-attribute 

diff_atol: float

The absolute tolerance for considering a difference "significant".

A difference is "significant" if abs(new - old) < self.diff_atol + self.diff_rtol * abs(old).

If a difference is not significant, it will not have the diff styles applied. Note that we still show both the rendered before and after values in the diff any time they differ, even if the difference is not significant. (If the rendered values are exactly the same, we only show the value once.)

If not provided, use 1e-6.

diff_rtol instance-attribute 

diff_rtol: float

The relative tolerance for considering a difference "significant".

See the description of diff_atol for more details about what makes a difference "significant".

If not provided, use 0.001 if all values are ints, otherwise 0.05.

diff_increase_style instance-attribute 

diff_increase_style: str

The style to apply to diffed values that have a significant increase.

See the description of diff_atol for more details about what makes a difference "significant".

If not provided, use green for scores and red for metrics. You can also use arbitrary rich styles, such as "bold red".

diff_decrease_style instance-attribute 

diff_decrease_style: str

The style to apply to diffed values that have significant decrease.

See the description of diff_atol for more details about what makes a difference "significant".

If not provided, use red for scores and green for metrics. You can also use arbitrary rich styles, such as "bold red".

ReportCaseRenderer dataclass

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
@dataclass(kw_only=True)
class ReportCaseRenderer:
    include_input: bool
    include_metadata: bool
    include_expected_output: bool
    include_output: bool
    include_scores: bool
    include_labels: bool
    include_metrics: bool
    include_assertions: bool
    include_reasons: bool
    include_durations: bool
    include_total_duration: bool
    include_error_message: bool
    include_error_stacktrace: bool
    include_evaluator_failures: bool

    input_renderer: _ValueRenderer
    metadata_renderer: _ValueRenderer
    output_renderer: _ValueRenderer
    score_renderers: Mapping[str, _NumberRenderer]
    label_renderers: Mapping[str, _ValueRenderer]
    metric_renderers: Mapping[str, _NumberRenderer]
    duration_renderer: _NumberRenderer

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def build_base_table(self, title: str) -> Table:
        """Build and return a Rich Table for the diff output."""
        table = Table(title=title, show_lines=True)
        table.add_column('Case ID', style='bold')
        if self.include_input:
            table.add_column('Inputs', overflow='fold')
        if self.include_metadata:
            table.add_column('Metadata', overflow='fold')
        if self.include_expected_output:
            table.add_column('Expected Output', overflow='fold')
        if self.include_output:
            table.add_column('Outputs', overflow='fold')
        if self.include_scores:
            table.add_column('Scores', overflow='fold')
        if self.include_labels:
            table.add_column('Labels', overflow='fold')
        if self.include_metrics:
            table.add_column('Metrics', overflow='fold')
        if self.include_assertions:
            table.add_column('Assertions', overflow='fold')
        if self.include_evaluator_failures:
            table.add_column('Evaluator Failures', overflow='fold')
        if self.include_durations:
            table.add_column('Durations' if self.include_total_duration else 'Duration', justify='right')
        return table

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def build_failures_table(self, title: str) -> Table:
        """Build and return a Rich Table for the failures output."""
        table = Table(title=title, show_lines=True)
        table.add_column('Case ID', style='bold')
        if self.include_input:
            table.add_column('Inputs', overflow='fold')
        if self.include_metadata:
            table.add_column('Metadata', overflow='fold')
        if self.include_expected_output:
            table.add_column('Expected Output', overflow='fold')
        if self.include_error_message:
            table.add_column('Error Message', overflow='fold')
        if self.include_error_stacktrace:
            table.add_column('Error Stacktrace', overflow='fold')
        return table

    def build_row(self, case: ReportCase) -> list[str]:
        """Build a table row for a single case."""
        row = [case.name]

        if self.include_input:
            row.append(self.input_renderer.render_value(None, case.inputs) or EMPTY_CELL_STR)

        if self.include_metadata:
            row.append(self.metadata_renderer.render_value(None, case.metadata) or EMPTY_CELL_STR)

        if self.include_expected_output:
            row.append(self.output_renderer.render_value(None, case.expected_output) or EMPTY_CELL_STR)

        if self.include_output:
            row.append(self.output_renderer.render_value(None, case.output) or EMPTY_CELL_STR)

        if self.include_scores:
            row.append(self._render_dict({k: v for k, v in case.scores.items()}, self.score_renderers))

        if self.include_labels:
            row.append(self._render_dict({k: v for k, v in case.labels.items()}, self.label_renderers))

        if self.include_metrics:
            row.append(self._render_dict(case.metrics, self.metric_renderers))

        if self.include_assertions:
            row.append(self._render_assertions(list(case.assertions.values())))

        if self.include_evaluator_failures:
            row.append(self._render_evaluator_failures(case.evaluator_failures))

        if self.include_durations:
            row.append(self._render_durations(case))

        return row

    def build_aggregate_row(self, aggregate: ReportCaseAggregate) -> list[str]:
        """Build a table row for an aggregated case."""
        row = [f'[b i]{aggregate.name}[/]']

        if self.include_input:
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_metadata:
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_expected_output:
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_output:
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_scores:
            row.append(self._render_dict(aggregate.scores, self.score_renderers))

        if self.include_labels:
            row.append(self._render_dict(aggregate.labels, self.label_renderers))

        if self.include_metrics:
            row.append(self._render_dict(aggregate.metrics, self.metric_renderers))

        if self.include_assertions:
            row.append(self._render_aggregate_assertions(aggregate.assertions))

        if self.include_evaluator_failures:
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_durations:
            row.append(self._render_durations(aggregate))

        return row

    def build_diff_row(
        self,
        new_case: ReportCase,
        baseline: ReportCase,
    ) -> list[str]:
        """Build a table row for a given case ID."""
        assert baseline.name == new_case.name, 'This should only be called for matching case IDs'
        row = [baseline.name]

        if self.include_input:  # pragma: no branch
            input_diff = self.input_renderer.render_diff(None, baseline.inputs, new_case.inputs) or EMPTY_CELL_STR
            row.append(input_diff)

        if self.include_metadata:  # pragma: no branch
            metadata_diff = (
                self.metadata_renderer.render_diff(None, baseline.metadata, new_case.metadata) or EMPTY_CELL_STR
            )
            row.append(metadata_diff)

        if self.include_expected_output:  # pragma: no branch
            expected_output_diff = (
                self.output_renderer.render_diff(None, baseline.expected_output, new_case.expected_output)
                or EMPTY_CELL_STR
            )
            row.append(expected_output_diff)

        if self.include_output:  # pragma: no branch
            output_diff = self.output_renderer.render_diff(None, baseline.output, new_case.output) or EMPTY_CELL_STR
            row.append(output_diff)

        if self.include_scores:  # pragma: no branch
            scores_diff = self._render_dicts_diff(
                {k: v.value for k, v in baseline.scores.items()},
                {k: v.value for k, v in new_case.scores.items()},
                self.score_renderers,
            )
            row.append(scores_diff)

        if self.include_labels:  # pragma: no branch
            labels_diff = self._render_dicts_diff(
                {k: v.value for k, v in baseline.labels.items()},
                {k: v.value for k, v in new_case.labels.items()},
                self.label_renderers,
            )
            row.append(labels_diff)

        if self.include_metrics:  # pragma: no branch
            metrics_diff = self._render_dicts_diff(baseline.metrics, new_case.metrics, self.metric_renderers)
            row.append(metrics_diff)

        if self.include_assertions:  # pragma: no branch
            assertions_diff = self._render_assertions_diff(
                list(baseline.assertions.values()), list(new_case.assertions.values())
            )
            row.append(assertions_diff)

        if self.include_evaluator_failures:  # pragma: no branch
            evaluator_failures_diff = self._render_evaluator_failures_diff(
                baseline.evaluator_failures, new_case.evaluator_failures
            )
            row.append(evaluator_failures_diff)

        if self.include_durations:  # pragma: no branch
            durations_diff = self._render_durations_diff(baseline, new_case)
            row.append(durations_diff)

        return row

    def build_diff_aggregate_row(
        self,
        new: ReportCaseAggregate,
        baseline: ReportCaseAggregate,
    ) -> list[str]:
        """Build a table row for a given case ID."""
        assert baseline.name == new.name, 'This should only be called for aggregates with matching names'
        row = [f'[b i]{baseline.name}[/]']

        if self.include_input:  # pragma: no branch
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_metadata:  # pragma: no branch
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_expected_output:  # pragma: no branch
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_output:  # pragma: no branch
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_scores:  # pragma: no branch
            scores_diff = self._render_dicts_diff(baseline.scores, new.scores, self.score_renderers)
            row.append(scores_diff)

        if self.include_labels:  # pragma: no branch
            labels_diff = self._render_dicts_diff(baseline.labels, new.labels, self.label_renderers)
            row.append(labels_diff)

        if self.include_metrics:  # pragma: no branch
            metrics_diff = self._render_dicts_diff(baseline.metrics, new.metrics, self.metric_renderers)
            row.append(metrics_diff)

        if self.include_assertions:  # pragma: no branch
            assertions_diff = self._render_aggregate_assertions_diff(baseline.assertions, new.assertions)
            row.append(assertions_diff)

        if self.include_evaluator_failures:  # pragma: no branch
            row.append(EMPTY_AGGREGATE_CELL_STR)

        if self.include_durations:  # pragma: no branch
            durations_diff = self._render_durations_diff(baseline, new)
            row.append(durations_diff)

        return row

    def build_failure_row(self, case: ReportCaseFailure) -> list[str]:
        """Build a table row for a single case failure."""
        row = [case.name]

        if self.include_input:
            row.append(self.input_renderer.render_value(None, case.inputs) or EMPTY_CELL_STR)

        if self.include_metadata:
            row.append(self.metadata_renderer.render_value(None, case.metadata) or EMPTY_CELL_STR)

        if self.include_expected_output:
            row.append(self.output_renderer.render_value(None, case.expected_output) or EMPTY_CELL_STR)

        if self.include_error_message:
            row.append(case.error_message or EMPTY_CELL_STR)

        if self.include_error_stacktrace:
            row.append(case.error_stacktrace or EMPTY_CELL_STR)

        return row

    def _render_durations(self, case: ReportCase | ReportCaseAggregate) -> str:
        """Build the diff string for a duration value."""
        case_durations: dict[str, float] = {'task': case.task_duration}
        if self.include_total_duration:
            case_durations['total'] = case.total_duration
        return self._render_dict(
            case_durations,
            {'task': self.duration_renderer, 'total': self.duration_renderer},
            include_names=self.include_total_duration,
        )

    def _render_durations_diff(
        self,
        base_case: ReportCase | ReportCaseAggregate,
        new_case: ReportCase | ReportCaseAggregate,
    ) -> str:
        """Build the diff string for a duration value."""
        base_case_durations: dict[str, float] = {'task': base_case.task_duration}
        new_case_durations: dict[str, float] = {'task': new_case.task_duration}
        if self.include_total_duration:  # pragma: no branch
            base_case_durations['total'] = base_case.total_duration
            new_case_durations['total'] = new_case.total_duration
        return self._render_dicts_diff(
            base_case_durations,
            new_case_durations,
            {'task': self.duration_renderer, 'total': self.duration_renderer},
            include_names=self.include_total_duration,
        )

    @staticmethod
    def _render_dicts_diff(
        baseline_dict: dict[str, T],
        new_dict: dict[str, T],
        renderers: Mapping[str, _AbstractRenderer[T]],
        *,
        include_names: bool = True,
    ) -> str:
        keys: set[str] = set()
        keys.update(baseline_dict.keys())
        keys.update(new_dict.keys())
        diff_lines: list[str] = []
        for key in sorted(keys):
            old_val = baseline_dict.get(key)
            new_val = new_dict.get(key)
            rendered = renderers[key].render_diff(key if include_names else None, old_val, new_val)
            diff_lines.append(rendered)
        return '\n'.join(diff_lines) if diff_lines else EMPTY_CELL_STR

    def _render_dict(
        self,
        case_dict: Mapping[str, EvaluationResult[T] | T],
        renderers: Mapping[str, _AbstractRenderer[T]],
        *,
        include_names: bool = True,
    ) -> str:
        diff_lines: list[str] = []
        for key, val in case_dict.items():
            value = cast(EvaluationResult[T], val).value if isinstance(val, EvaluationResult) else val
            rendered = renderers[key].render_value(key if include_names else None, value)
            if self.include_reasons and isinstance(val, EvaluationResult) and (reason := val.reason):
                rendered += f'\n  Reason: {reason}\n'
            diff_lines.append(rendered)
        return '\n'.join(diff_lines) if diff_lines else EMPTY_CELL_STR

    def _render_assertions(
        self,
        assertions: list[EvaluationResult[bool]],
    ) -> str:
        if not assertions:
            return EMPTY_CELL_STR
        lines: list[str] = []
        for a in assertions:
            line = '[green]✔[/]' if a.value else '[red]✗[/]'
            if self.include_reasons:
                line = f'{a.name}: {line}\n'
                line = f'{line}  Reason: {a.reason}\n\n' if a.reason else line
            lines.append(line)
        return ''.join(lines)

    @staticmethod
    def _render_aggregate_assertions(
        assertions: float | None,
    ) -> str:
        return (
            default_render_percentage(assertions) + ' [green]✔[/]'
            if assertions is not None
            else EMPTY_AGGREGATE_CELL_STR
        )

    @staticmethod
    def _render_assertions_diff(
        assertions: list[EvaluationResult[bool]], new_assertions: list[EvaluationResult[bool]]
    ) -> str:
        if not assertions and not new_assertions:  # pragma: no cover
            return EMPTY_CELL_STR

        old = ''.join(['[green]✔[/]' if a.value else '[red]✗[/]' for a in assertions])
        new = ''.join(['[green]✔[/]' if a.value else '[red]✗[/]' for a in new_assertions])
        return old if old == new else f'{old} → {new}'

    @staticmethod
    def _render_aggregate_assertions_diff(
        baseline: float | None,
        new: float | None,
    ) -> str:
        if baseline is None and new is None:  # pragma: no cover
            return EMPTY_AGGREGATE_CELL_STR
        rendered_baseline = (
            default_render_percentage(baseline) + ' [green]✔[/]' if baseline is not None else EMPTY_CELL_STR
        )
        rendered_new = default_render_percentage(new) + ' [green]✔[/]' if new is not None else EMPTY_CELL_STR
        return rendered_new if rendered_baseline == rendered_new else f'{rendered_baseline} → {rendered_new}'

    def _render_evaluator_failures(
        self,
        failures: list[EvaluatorFailure],
    ) -> str:
        if not failures:
            return EMPTY_CELL_STR  # pragma: no cover
        lines: list[str] = []
        for failure in failures:
            line = f'[red]{failure.name}[/]'
            if failure.error_message:
                line += f': {failure.error_message}'
            lines.append(line)
        return '\n'.join(lines)

    def _render_evaluator_failures_diff(
        self,
        baseline_failures: list[EvaluatorFailure],
        new_failures: list[EvaluatorFailure],
    ) -> str:
        baseline_str = self._render_evaluator_failures(baseline_failures)
        new_str = self._render_evaluator_failures(new_failures)
        if baseline_str == new_str:
            return baseline_str  # pragma: no cover
        return f'{baseline_str}\n→\n{new_str}'

build_base_table 

build_base_table(title: str) -> Table

Build and return a Rich Table for the diff output.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
def build_base_table(self, title: str) -> Table:
    """Build and return a Rich Table for the diff output."""
    table = Table(title=title, show_lines=True)
    table.add_column('Case ID', style='bold')
    if self.include_input:
        table.add_column('Inputs', overflow='fold')
    if self.include_metadata:
        table.add_column('Metadata', overflow='fold')
    if self.include_expected_output:
        table.add_column('Expected Output', overflow='fold')
    if self.include_output:
        table.add_column('Outputs', overflow='fold')
    if self.include_scores:
        table.add_column('Scores', overflow='fold')
    if self.include_labels:
        table.add_column('Labels', overflow='fold')
    if self.include_metrics:
        table.add_column('Metrics', overflow='fold')
    if self.include_assertions:
        table.add_column('Assertions', overflow='fold')
    if self.include_evaluator_failures:
        table.add_column('Evaluator Failures', overflow='fold')
    if self.include_durations:
        table.add_column('Durations' if self.include_total_duration else 'Duration', justify='right')
    return table

build_failures_table 

build_failures_table(title: str) -> Table

Build and return a Rich Table for the failures output.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
def build_failures_table(self, title: str) -> Table:
    """Build and return a Rich Table for the failures output."""
    table = Table(title=title, show_lines=True)
    table.add_column('Case ID', style='bold')
    if self.include_input:
        table.add_column('Inputs', overflow='fold')
    if self.include_metadata:
        table.add_column('Metadata', overflow='fold')
    if self.include_expected_output:
        table.add_column('Expected Output', overflow='fold')
    if self.include_error_message:
        table.add_column('Error Message', overflow='fold')
    if self.include_error_stacktrace:
        table.add_column('Error Stacktrace', overflow='fold')
    return table

build_row 

build_row(case: ReportCase) -> list[str]

Build a table row for a single case.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
def build_row(self, case: ReportCase) -> list[str]:
    """Build a table row for a single case."""
    row = [case.name]

    if self.include_input:
        row.append(self.input_renderer.render_value(None, case.inputs) or EMPTY_CELL_STR)

    if self.include_metadata:
        row.append(self.metadata_renderer.render_value(None, case.metadata) or EMPTY_CELL_STR)

    if self.include_expected_output:
        row.append(self.output_renderer.render_value(None, case.expected_output) or EMPTY_CELL_STR)

    if self.include_output:
        row.append(self.output_renderer.render_value(None, case.output) or EMPTY_CELL_STR)

    if self.include_scores:
        row.append(self._render_dict({k: v for k, v in case.scores.items()}, self.score_renderers))

    if self.include_labels:
        row.append(self._render_dict({k: v for k, v in case.labels.items()}, self.label_renderers))

    if self.include_metrics:
        row.append(self._render_dict(case.metrics, self.metric_renderers))

    if self.include_assertions:
        row.append(self._render_assertions(list(case.assertions.values())))

    if self.include_evaluator_failures:
        row.append(self._render_evaluator_failures(case.evaluator_failures))

    if self.include_durations:
        row.append(self._render_durations(case))

    return row

build_aggregate_row 

build_aggregate_row(
    aggregate: ReportCaseAggregate,
) -> list[str]

Build a table row for an aggregated case.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
def build_aggregate_row(self, aggregate: ReportCaseAggregate) -> list[str]:
    """Build a table row for an aggregated case."""
    row = [f'[b i]{aggregate.name}[/]']

    if self.include_input:
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_metadata:
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_expected_output:
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_output:
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_scores:
        row.append(self._render_dict(aggregate.scores, self.score_renderers))

    if self.include_labels:
        row.append(self._render_dict(aggregate.labels, self.label_renderers))

    if self.include_metrics:
        row.append(self._render_dict(aggregate.metrics, self.metric_renderers))

    if self.include_assertions:
        row.append(self._render_aggregate_assertions(aggregate.assertions))

    if self.include_evaluator_failures:
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_durations:
        row.append(self._render_durations(aggregate))

    return row

build_diff_row 

build_diff_row(
    new_case: ReportCase, baseline: ReportCase
) -> list[str]

Build a table row for a given case ID.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
def build_diff_row(
    self,
    new_case: ReportCase,
    baseline: ReportCase,
) -> list[str]:
    """Build a table row for a given case ID."""
    assert baseline.name == new_case.name, 'This should only be called for matching case IDs'
    row = [baseline.name]

    if self.include_input:  # pragma: no branch
        input_diff = self.input_renderer.render_diff(None, baseline.inputs, new_case.inputs) or EMPTY_CELL_STR
        row.append(input_diff)

    if self.include_metadata:  # pragma: no branch
        metadata_diff = (
            self.metadata_renderer.render_diff(None, baseline.metadata, new_case.metadata) or EMPTY_CELL_STR
        )
        row.append(metadata_diff)

    if self.include_expected_output:  # pragma: no branch
        expected_output_diff = (
            self.output_renderer.render_diff(None, baseline.expected_output, new_case.expected_output)
            or EMPTY_CELL_STR
        )
        row.append(expected_output_diff)

    if self.include_output:  # pragma: no branch
        output_diff = self.output_renderer.render_diff(None, baseline.output, new_case.output) or EMPTY_CELL_STR
        row.append(output_diff)

    if self.include_scores:  # pragma: no branch
        scores_diff = self._render_dicts_diff(
            {k: v.value for k, v in baseline.scores.items()},
            {k: v.value for k, v in new_case.scores.items()},
            self.score_renderers,
        )
        row.append(scores_diff)

    if self.include_labels:  # pragma: no branch
        labels_diff = self._render_dicts_diff(
            {k: v.value for k, v in baseline.labels.items()},
            {k: v.value for k, v in new_case.labels.items()},
            self.label_renderers,
        )
        row.append(labels_diff)

    if self.include_metrics:  # pragma: no branch
        metrics_diff = self._render_dicts_diff(baseline.metrics, new_case.metrics, self.metric_renderers)
        row.append(metrics_diff)

    if self.include_assertions:  # pragma: no branch
        assertions_diff = self._render_assertions_diff(
            list(baseline.assertions.values()), list(new_case.assertions.values())
        )
        row.append(assertions_diff)

    if self.include_evaluator_failures:  # pragma: no branch
        evaluator_failures_diff = self._render_evaluator_failures_diff(
            baseline.evaluator_failures, new_case.evaluator_failures
        )
        row.append(evaluator_failures_diff)

    if self.include_durations:  # pragma: no branch
        durations_diff = self._render_durations_diff(baseline, new_case)
        row.append(durations_diff)

    return row

build_diff_aggregate_row 

build_diff_aggregate_row(
    new: ReportCaseAggregate, baseline: ReportCaseAggregate
) -> list[str]

Build a table row for a given case ID.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
def build_diff_aggregate_row(
    self,
    new: ReportCaseAggregate,
    baseline: ReportCaseAggregate,
) -> list[str]:
    """Build a table row for a given case ID."""
    assert baseline.name == new.name, 'This should only be called for aggregates with matching names'
    row = [f'[b i]{baseline.name}[/]']

    if self.include_input:  # pragma: no branch
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_metadata:  # pragma: no branch
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_expected_output:  # pragma: no branch
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_output:  # pragma: no branch
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_scores:  # pragma: no branch
        scores_diff = self._render_dicts_diff(baseline.scores, new.scores, self.score_renderers)
        row.append(scores_diff)

    if self.include_labels:  # pragma: no branch
        labels_diff = self._render_dicts_diff(baseline.labels, new.labels, self.label_renderers)
        row.append(labels_diff)

    if self.include_metrics:  # pragma: no branch
        metrics_diff = self._render_dicts_diff(baseline.metrics, new.metrics, self.metric_renderers)
        row.append(metrics_diff)

    if self.include_assertions:  # pragma: no branch
        assertions_diff = self._render_aggregate_assertions_diff(baseline.assertions, new.assertions)
        row.append(assertions_diff)

    if self.include_evaluator_failures:  # pragma: no branch
        row.append(EMPTY_AGGREGATE_CELL_STR)

    if self.include_durations:  # pragma: no branch
        durations_diff = self._render_durations_diff(baseline, new)
        row.append(durations_diff)

    return row

build_failure_row 

build_failure_row(case: ReportCaseFailure) -> list[str]

Build a table row for a single case failure.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
def build_failure_row(self, case: ReportCaseFailure) -> list[str]:
    """Build a table row for a single case failure."""
    row = [case.name]

    if self.include_input:
        row.append(self.input_renderer.render_value(None, case.inputs) or EMPTY_CELL_STR)

    if self.include_metadata:
        row.append(self.metadata_renderer.render_value(None, case.metadata) or EMPTY_CELL_STR)

    if self.include_expected_output:
        row.append(self.output_renderer.render_value(None, case.expected_output) or EMPTY_CELL_STR)

    if self.include_error_message:
        row.append(case.error_message or EMPTY_CELL_STR)

    if self.include_error_stacktrace:
        row.append(case.error_stacktrace or EMPTY_CELL_STR)

    return row

EvaluationRenderer dataclass

A class for rendering an EvalReport or the diff between two EvalReports.

Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
@dataclass(kw_only=True)
class EvaluationRenderer:
    """A class for rendering an EvalReport or the diff between two EvalReports."""

    # Columns to include
    include_input: bool
    include_metadata: bool
    include_expected_output: bool
    include_output: bool
    include_durations: bool
    include_total_duration: bool

    # Rows to include
    include_removed_cases: bool
    include_averages: bool

    input_config: RenderValueConfig
    metadata_config: RenderValueConfig
    output_config: RenderValueConfig
    score_configs: dict[str, RenderNumberConfig]
    label_configs: dict[str, RenderValueConfig]
    metric_configs: dict[str, RenderNumberConfig]
    duration_config: RenderNumberConfig

    # Data to include
    include_reasons: bool  # only applies to reports, not to diffs

    include_error_message: bool
    include_error_stacktrace: bool
    include_evaluator_failures: bool

    def include_scores(self, report: EvaluationReport, baseline: EvaluationReport | None = None):
        return any(case.scores for case in self._all_cases(report, baseline))

    def include_labels(self, report: EvaluationReport, baseline: EvaluationReport | None = None):
        return any(case.labels for case in self._all_cases(report, baseline))

    def include_metrics(self, report: EvaluationReport, baseline: EvaluationReport | None = None):
        return any(case.metrics for case in self._all_cases(report, baseline))

    def include_assertions(self, report: EvaluationReport, baseline: EvaluationReport | None = None):
        return any(case.assertions for case in self._all_cases(report, baseline))

    def include_evaluator_failures_column(self, report: EvaluationReport, baseline: EvaluationReport | None = None):
        return self.include_evaluator_failures and any(
            case.evaluator_failures for case in self._all_cases(report, baseline)
        )

    def _all_cases(self, report: EvaluationReport, baseline: EvaluationReport | None) -> list[ReportCase]:
        if not baseline:
            return report.cases
        else:
            return report.cases + self._baseline_cases_to_include(report, baseline)

    def _baseline_cases_to_include(self, report: EvaluationReport, baseline: EvaluationReport) -> list[ReportCase]:
        if self.include_removed_cases:
            return baseline.cases
        report_case_names = {case.name for case in report.cases}
        return [case for case in baseline.cases if case.name in report_case_names]

    def _get_case_renderer(
        self, report: EvaluationReport, baseline: EvaluationReport | None = None
    ) -> ReportCaseRenderer:
        input_renderer = _ValueRenderer.from_config(self.input_config)
        metadata_renderer = _ValueRenderer.from_config(self.metadata_config)
        output_renderer = _ValueRenderer.from_config(self.output_config)
        score_renderers = self._infer_score_renderers(report, baseline)
        label_renderers = self._infer_label_renderers(report, baseline)
        metric_renderers = self._infer_metric_renderers(report, baseline)
        duration_renderer = _NumberRenderer.infer_from_config(
            self.duration_config, 'duration', [x.task_duration for x in self._all_cases(report, baseline)]
        )

        return ReportCaseRenderer(
            include_input=self.include_input,
            include_metadata=self.include_metadata,
            include_expected_output=self.include_expected_output,
            include_output=self.include_output,
            include_scores=self.include_scores(report, baseline),
            include_labels=self.include_labels(report, baseline),
            include_metrics=self.include_metrics(report, baseline),
            include_assertions=self.include_assertions(report, baseline),
            include_reasons=self.include_reasons,
            include_durations=self.include_durations,
            include_total_duration=self.include_total_duration,
            include_error_message=self.include_error_message,
            include_error_stacktrace=self.include_error_stacktrace,
            include_evaluator_failures=self.include_evaluator_failures_column(report, baseline),
            input_renderer=input_renderer,
            metadata_renderer=metadata_renderer,
            output_renderer=output_renderer,
            score_renderers=score_renderers,
            label_renderers=label_renderers,
            metric_renderers=metric_renderers,
            duration_renderer=duration_renderer,
        )

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def build_table(self, report: EvaluationReport, *, with_title: bool = True) -> Table:
        """Build a table for the report.

        Args:
            report: The evaluation report to render
            with_title: Whether to include the title in the table (default True)

        Returns:
            A Rich Table object
        """
        case_renderer = self._get_case_renderer(report)

        title = f'Evaluation Summary: {report.name}' if with_title else ''
        table = case_renderer.build_base_table(title)

        for case in report.cases:
            table.add_row(*case_renderer.build_row(case))

        if self.include_averages:  # pragma: no branch
            average = report.averages()
            if average:  # pragma: no branch
                table.add_row(*case_renderer.build_aggregate_row(average))

        return table

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def build_diff_table(
        self, report: EvaluationReport, baseline: EvaluationReport, *, with_title: bool = True
    ) -> Table:
        """Build a diff table comparing report to baseline.

        Args:
            report: The evaluation report to compare
            baseline: The baseline report to compare against
            with_title: Whether to include the title in the table (default True)

        Returns:
            A Rich Table object
        """
        report_cases = report.cases
        baseline_cases = self._baseline_cases_to_include(report, baseline)

        report_cases_by_id = {case.name: case for case in report_cases}
        baseline_cases_by_id = {case.name: case for case in baseline_cases}

        diff_cases: list[tuple[ReportCase, ReportCase]] = []
        removed_cases: list[ReportCase] = []
        added_cases: list[ReportCase] = []

        for case_id in sorted(set(baseline_cases_by_id.keys()) | set(report_cases_by_id.keys())):
            maybe_baseline_case = baseline_cases_by_id.get(case_id)
            maybe_report_case = report_cases_by_id.get(case_id)
            if maybe_baseline_case and maybe_report_case:
                diff_cases.append((maybe_baseline_case, maybe_report_case))
            elif maybe_baseline_case:
                removed_cases.append(maybe_baseline_case)
            elif maybe_report_case:
                added_cases.append(maybe_report_case)
            else:  # pragma: no cover
                assert False, 'This should be unreachable'

        case_renderer = self._get_case_renderer(report, baseline)
        diff_name = baseline.name if baseline.name == report.name else f'{baseline.name} → {report.name}'

        title = f'Evaluation Diff: {diff_name}' if with_title else ''
        table = case_renderer.build_base_table(title)

        for baseline_case, new_case in diff_cases:
            table.add_row(*case_renderer.build_diff_row(new_case, baseline_case))
        for case in added_cases:
            row = case_renderer.build_row(case)
            row[0] = f'[green]+ Added Case[/]\n{row[0]}'
            table.add_row(*row)
        for case in removed_cases:
            row = case_renderer.build_row(case)
            row[0] = f'[red]- Removed Case[/]\n{row[0]}'
            table.add_row(*row)

        if self.include_averages:  # pragma: no branch
            # Use flat averaging for both sides to keep the diff symmetric.
            # baseline_cases is already filtered to only cases matching the report.
            # Note: for multi-run reports, this differs from build_table which uses two-level
            # aggregation via report.averages(). In practice the results are identical when all
            # runs succeed (equal group sizes), and only diverge with partial failures within a
            # group — a rare edge case. We can revisit if users report confusing behavior.
            report_average = ReportCaseAggregate.average(report_cases) if report_cases else None
            baseline_average = ReportCaseAggregate.average(baseline_cases) if baseline_cases else None
            if report_average and baseline_average:  # pragma: no branch
                table.add_row(*case_renderer.build_diff_aggregate_row(report_average, baseline_average))

        return table

    # TODO(DavidM): in v2, change the return type here to RenderableType
    def build_failures_table(self, report: EvaluationReport) -> Table:
        case_renderer = self._get_case_renderer(report)
        table = case_renderer.build_failures_table('Case Failures')
        for case in report.failures:
            table.add_row(*case_renderer.build_failure_row(case))

        return table

    def _infer_score_renderers(
        self, report: EvaluationReport, baseline: EvaluationReport | None
    ) -> dict[str, _NumberRenderer]:
        all_cases = self._all_cases(report, baseline)

        values_by_name: dict[str, list[float | int]] = {}
        for case in all_cases:
            for k, score in case.scores.items():
                values_by_name.setdefault(k, []).append(score.value)

        all_renderers: dict[str, _NumberRenderer] = {}
        for name, values in values_by_name.items():
            merged_config = _DEFAULT_NUMBER_CONFIG.copy()
            merged_config.update(self.score_configs.get(name, {}))
            all_renderers[name] = _NumberRenderer.infer_from_config(merged_config, 'score', values)
        return all_renderers

    def _infer_label_renderers(
        self, report: EvaluationReport, baseline: EvaluationReport | None
    ) -> dict[str, _ValueRenderer]:
        all_cases = self._all_cases(report, baseline)
        all_names: set[str] = set()
        for case in all_cases:
            for k in case.labels:
                all_names.add(k)

        all_renderers: dict[str, _ValueRenderer] = {}
        for name in all_names:
            merged_config = _DEFAULT_VALUE_CONFIG.copy()
            merged_config.update(self.label_configs.get(name, {}))
            all_renderers[name] = _ValueRenderer.from_config(merged_config)
        return all_renderers

    def _infer_metric_renderers(
        self, report: EvaluationReport, baseline: EvaluationReport | None
    ) -> dict[str, _NumberRenderer]:
        all_cases = self._all_cases(report, baseline)

        values_by_name: dict[str, list[float | int]] = {}
        for case in all_cases:
            for k, v in case.metrics.items():
                values_by_name.setdefault(k, []).append(v)

        all_renderers: dict[str, _NumberRenderer] = {}
        for name, values in values_by_name.items():
            merged_config = _DEFAULT_NUMBER_CONFIG.copy()
            merged_config.update(self.metric_configs.get(name, {}))
            all_renderers[name] = _NumberRenderer.infer_from_config(merged_config, 'metric', values)
        return all_renderers

    def _infer_duration_renderer(
        self, report: EvaluationReport, baseline: EvaluationReport | None
    ) -> _NumberRenderer:  # pragma: no cover
        all_cases = self._all_cases(report, baseline)
        all_durations = [x.task_duration for x in all_cases]
        if self.include_total_duration:
            all_durations += [x.total_duration for x in all_cases]
        return _NumberRenderer.infer_from_config(self.duration_config, 'duration', all_durations)

build_table 

build_table(
    report: EvaluationReport, *, with_title: bool = True
) -> Table

Build a table for the report.

Parameters:

Name Type Description Default
report EvaluationReport

The evaluation report to render

 required
with_title bool

Whether to include the title in the table (default True)

 True

Returns:

Type Description
Table

A Rich Table object
Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
def build_table(self, report: EvaluationReport, *, with_title: bool = True) -> Table:
    """Build a table for the report.

    Args:
        report: The evaluation report to render
        with_title: Whether to include the title in the table (default True)

    Returns:
        A Rich Table object
    """
    case_renderer = self._get_case_renderer(report)

    title = f'Evaluation Summary: {report.name}' if with_title else ''
    table = case_renderer.build_base_table(title)

    for case in report.cases:
        table.add_row(*case_renderer.build_row(case))

    if self.include_averages:  # pragma: no branch
        average = report.averages()
        if average:  # pragma: no branch
            table.add_row(*case_renderer.build_aggregate_row(average))

    return table

build_diff_table 

build_diff_table(
    report: EvaluationReport,
    baseline: EvaluationReport,
    *,
    with_title: bool = True
) -> Table

Build a diff table comparing report to baseline.

Parameters:

Name Type Description Default
report EvaluationReport

The evaluation report to compare

 required
baseline EvaluationReport

The baseline report to compare against

 required
with_title bool

Whether to include the title in the table (default True)

 True

Returns:

Type Description
Table

A Rich Table object
Source code in pydantic_evals/pydantic_evals/reporting/__init__.py 
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
def build_diff_table(
    self, report: EvaluationReport, baseline: EvaluationReport, *, with_title: bool = True
) -> Table:
    """Build a diff table comparing report to baseline.

    Args:
        report: The evaluation report to compare
        baseline: The baseline report to compare against
        with_title: Whether to include the title in the table (default True)

    Returns:
        A Rich Table object
    """
    report_cases = report.cases
    baseline_cases = self._baseline_cases_to_include(report, baseline)

    report_cases_by_id = {case.name: case for case in report_cases}
    baseline_cases_by_id = {case.name: case for case in baseline_cases}

    diff_cases: list[tuple[ReportCase, ReportCase]] = []
    removed_cases: list[ReportCase] = []
    added_cases: list[ReportCase] = []

    for case_id in sorted(set(baseline_cases_by_id.keys()) | set(report_cases_by_id.keys())):
        maybe_baseline_case = baseline_cases_by_id.get(case_id)
        maybe_report_case = report_cases_by_id.get(case_id)
        if maybe_baseline_case and maybe_report_case:
            diff_cases.append((maybe_baseline_case, maybe_report_case))
        elif maybe_baseline_case:
            removed_cases.append(maybe_baseline_case)
        elif maybe_report_case:
            added_cases.append(maybe_report_case)
        else:  # pragma: no cover
            assert False, 'This should be unreachable'

    case_renderer = self._get_case_renderer(report, baseline)
    diff_name = baseline.name if baseline.name == report.name else f'{baseline.name} → {report.name}'

    title = f'Evaluation Diff: {diff_name}' if with_title else ''
    table = case_renderer.build_base_table(title)

    for baseline_case, new_case in diff_cases:
        table.add_row(*case_renderer.build_diff_row(new_case, baseline_case))
    for case in added_cases:
        row = case_renderer.build_row(case)
        row[0] = f'[green]+ Added Case[/]\n{row[0]}'
        table.add_row(*row)
    for case in removed_cases:
        row = case_renderer.build_row(case)
        row[0] = f'[red]- Removed Case[/]\n{row[0]}'
        table.add_row(*row)

    if self.include_averages:  # pragma: no branch
        # Use flat averaging for both sides to keep the diff symmetric.
        # baseline_cases is already filtered to only cases matching the report.
        # Note: for multi-run reports, this differs from build_table which uses two-level
        # aggregation via report.averages(). In practice the results are identical when all
        # runs succeed (equal group sizes), and only diverge with partial failures within a
        # group — a rare edge case. We can revisit if users report confusing behavior.
        report_average = ReportCaseAggregate.average(report_cases) if report_cases else None
        baseline_average = ReportCaseAggregate.average(baseline_cases) if baseline_cases else None
        if report_average and baseline_average:  # pragma: no branch
            table.add_row(*case_renderer.build_diff_aggregate_row(report_average, baseline_average))

    return table