def evaluate(
suite: Annotated[
str,
typer.Argument(help="EvalSuite reference in module:attribute form."),
],
tag: Annotated[
list[str] | None,
typer.Option("--tag", help="Require this scenario tag; repeatable."),
] = None,
profile: Annotated[
EvalProfile | None,
typer.Option("--profile", help="Select fast, integration, or live infrastructure."),
] = None,
trials: Annotated[
int | None,
typer.Option("--trials", min=1, help="Override trial count for selected scenarios."),
] = None,
pass_threshold: Annotated[
float | None,
typer.Option(
"--pass-threshold",
min=0.000001,
max=1.0,
help="Override the required passing-trial fraction.",
),
] = None,
json_output: Annotated[
Path | None,
typer.Option("--json", help="Write the canonical JSON report."),
] = None,
junit_output: Annotated[
Path | None,
typer.Option("--junit", help="Write a payload-free JUnit report."),
] = None,
) -> None:
"""Run a project's evidence-based agent regression suite."""
try:
definition = _load(suite)
required_tags = frozenset((*tuple(tag or ()), *((profile.value,) if profile else ())))
target_factory = definition.target_for(profile.value if profile else None)
scenarios, resolutions = asyncio.run(
resolve_suite_scenarios(
definition, target_factory, profile.value if profile else ""
)
)
if trials is not None or pass_threshold is not None:
scenarios = tuple(
replace(
scenario,
policy=TrialPolicy(
trials=trials or scenario.policy.trials,
pass_threshold=(
pass_threshold
if pass_threshold is not None
else scenario.policy.pass_threshold
),
timeout_seconds=scenario.policy.timeout_seconds,
),
)
for scenario in scenarios
)
report = asyncio.run(
run_suite(
scenarios,
target_factory,
tags=required_tags,
)
)
except (ImportError, AttributeError, TypeError, ValueError) as exc:
typer.echo(f"invalid eval suite: {exc}", err=True)
raise typer.Exit(2) from exc
rendered = json_report(report, resolutions)
if json_output is not None:
json_output.write_text(rendered, encoding="utf-8")
if junit_output is not None:
junit_output.write_text(junit_report(report, resolutions), encoding="utf-8")
typer.echo(
f"{report.status.value}: "
f"{sum(row.status is EvalStatus.PASSED for row in report.scenarios)}/"
f"{len(report.scenarios)} scenarios passed"
+ (
f"; {sum(not row.applicable for row in resolutions)} packs not applicable"
if resolutions
else ""
)
)
raise typer.Exit(EXIT_CODES[report.status])