Skip to content

symfonic.evals.tool_journeys

tool_journeys

Reusable SC-13 journey for a routed application tool.

tool_behavior_journey 

tool_behavior_journey() -> Scenario

Exercise execution, avoidance, schema rejection and failure containment.

Source code in src/symfonic/evals/tool_journeys.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def tool_behavior_journey() -> Scenario:
    """Exercise execution, avoidance, schema rejection and failure containment."""
    return Scenario(
        "scaffold-tool-behavior",
        (
            EvalStep(
                CORRECT_TOOL_PROMPT,
                (
                    ResponseEquals("Ada greeted"),
                    ToolCalled("greet_user", arguments={"name": "Ada"}),
                    ToolSucceeded("greet_user", arguments={"name": "Ada"}),
                    ToolFailed("greet_user", times=0),
                    StageObserved("tools.routing"),
                ),
            ),
            EvalStep(
                IRRELEVANT_TOOL_PROMPT,
                (
                    ResponseEquals("Chapter 5 explained directly"),
                    ToolCalled("greet_user", times=0),
                    ToolSucceeded("greet_user", times=0),
                    StageObserved("tools.routing"),
                ),
            ),
            EvalStep(
                MALFORMED_TOOL_PROMPT,
                (
                    ResponseEquals("Malformed greeting rejected"),
                    ToolCalled("greet_user"),
                    ToolSucceeded("greet_user", times=0),
                    ToolFailed("greet_user"),
                    StageObserved("tools.routing"),
                ),
            ),
            EvalStep(
                FAILING_TOOL_PROMPT,
                (
                    ResponseEquals("Tool failure contained"),
                    ToolCalled("failing_tool"),
                    ToolSucceeded("failing_tool", times=0),
                    ToolFailed("failing_tool"),
                    StageObserved("tools.routing"),
                ),
            ),
        ),
        tags=frozenset({"tools", "scaffold", "integration"}),
    )