Skip to content

symfonic.evals.reporters

reporters

Payload-safe JSON and JUnit reports for evaluation results.

json_report 

json_report(report: EvalReport, resolutions: tuple[PackResolution, ...] = ()) -> str

Serialize a report as deterministic formatted JSON.

Source code in src/symfonic/evals/reporters.py 
58
59
60
61
62
def json_report(
    report: EvalReport, resolutions: tuple[PackResolution, ...] = ()
) -> str:
    """Serialize a report as deterministic formatted JSON."""
    return json.dumps(report_dict(report, resolutions), indent=2, sort_keys=True) + "\n"

junit_report 

junit_report(report: EvalReport, resolutions: tuple[PackResolution, ...] = ()) -> str

Serialize scenarios as JUnit test cases without prompt or response payloads.

Source code in src/symfonic/evals/reporters.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def junit_report(
    report: EvalReport, resolutions: tuple[PackResolution, ...] = ()
) -> str:
    """Serialize scenarios as JUnit test cases without prompt or response payloads."""
    root = Element(
        "testsuite",
        name="symfonic-evals",
        tests=str(len(report.scenarios) + sum(not row.applicable for row in resolutions)),
        failures=str(sum(row.status is EvalStatus.FAILED for row in report.scenarios)),
        errors=str(sum(row.status is EvalStatus.ERROR for row in report.scenarios)),
        skipped=str(
            sum(row.status is EvalStatus.UNAVAILABLE for row in report.scenarios)
            + sum(not row.applicable for row in resolutions)
        ),
    )
    for scenario in report.scenarios:
        case = SubElement(root, "testcase", name=scenario.name, classname="symfonic.evals")
        if scenario.status is EvalStatus.FAILED:
            SubElement(case, "failure", message="behaviour threshold was not met")
        elif scenario.status is EvalStatus.ERROR:
            SubElement(case, "error", message="evaluation execution failed")
        elif scenario.status is EvalStatus.UNAVAILABLE:
            SubElement(case, "skipped", message="evaluation infrastructure unavailable")
    for resolution in resolutions:
        if resolution.applicable:
            continue
        case = SubElement(
            root,
            "testcase",
            name=f"pack:{resolution.pack}",
            classname="symfonic.evals.applicability",
        )
        SubElement(case, "skipped", message=resolution.reason)
    return tostring(root, encoding="unicode", xml_declaration=True) + "\n"

report_dict 

report_dict(report: EvalReport, resolutions: tuple[PackResolution, ...] = ()) -> dict[str, object]

Return the stable, JSON-safe report representation.

Source code in src/symfonic/evals/reporters.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def report_dict(
    report: EvalReport, resolutions: tuple[PackResolution, ...] = ()
) -> dict[str, object]:
    """Return the stable, JSON-safe report representation."""
    scenarios: list[dict[str, object]] = []
    for scenario in report.scenarios:
        trials: list[dict[str, object]] = []
        for trial in scenario.trials:
            trials.append(
                {
                    "index": trial.index,
                    "status": trial.status.value,
                    "duration_ms": trial.duration_ms,
                    "steps": [
                        {
                            "duration_ms": step.duration_ms,
                            "assertions": [
                                {"name": row.name, "passed": row.passed}
                                for row in step.assertions
                            ],
                        }
                        for step in trial.steps
                    ],
                }
            )
        scenarios.append(
            {
                "name": scenario.name,
                "status": scenario.status.value,
                "required_passes": scenario.required_passes,
                "passed_trials": scenario.passed_trials,
                "tags": sorted(scenario.tags),
                "trials": trials,
            }
        )
    rendered = {
        "schema_version": 1,
        "framework_version": report.framework_version,
        "status": report.status.value,
        "scenarios": scenarios,
    }
    if resolutions:
        rendered["applicability"] = applicability_report(resolutions)
    return rendered