resolve_suite_scenarios(suite: EvalSuite, target_factory: Callable[[], Any], profile: str) -> tuple[tuple[Scenario, ...], tuple[PackResolution, ...]]
Resolve packs from a disposable instance of the selected real target.
The probe is closed before any trial starts. Trials still receive a fresh
target from target_factory; probing can therefore neither seed memory
nor make a continuity assertion pass from state created outside its trial.
Source code in src/symfonic/evals/selection.py
| async def resolve_suite_scenarios(
suite: EvalSuite,
target_factory: Callable[[], Any],
profile: str,
) -> tuple[tuple[Scenario, ...], tuple[PackResolution, ...]]:
"""Resolve packs from a disposable instance of the selected real target.
The probe is closed before any trial starts. Trials still receive a fresh
target from ``target_factory``; probing can therefore neither seed memory
nor make a continuity assertion pass from state created outside its trial.
"""
if suite.pack_factory is None:
return suite.scenarios, ()
target = target_factory()
try:
probe = getattr(target, "capability_evidence", None)
if probe is None:
raise TypeError(
"a suite with optional packs requires a target exposing "
"capability_evidence()"
)
evidence = probe()
if inspect.isawaitable(evidence):
evidence = await evidence
if not isinstance(evidence, CapabilityEvidence):
raise TypeError("capability_evidence() must return CapabilityEvidence")
resolutions = tuple(suite.pack_factory(evidence, profile))
if any(not isinstance(row, PackResolution) for row in resolutions):
raise TypeError("pack_factory must return PackResolution values")
return suite.scenarios + applicable_scenarios(resolutions), resolutions
finally:
await _close(target)
|