Skip to content

symfonic.evals.book_evidence

book_evidence

Build normalized Glass Harbor evidence from the response actually returned.

The eval prompts request compact machine-readable annotations. Parsing those annotations here makes the grounding verdict independent of a target claiming that its own prose was safe. Reports still receive only assertion verdicts; the content-bearing mapping remains opt-in target evidence.

build_book_response_evidence 

build_book_response_evidence(response: str, fixture: BookFixture | None = None) -> Mapping[str, object]

Parse response annotations and grade their normalized propositions.

Supported annotations are [assert:fact=value], [unsupported:negative-fact-id] and [input:value]. Duplicate fact declarations or malformed annotation prefixes invalidate the evidence.

Source code in src/symfonic/evals/book_evidence.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
def build_book_response_evidence(
    response: str,
    fixture: BookFixture | None = None,
) -> Mapping[str, object]:
    """Parse response annotations and grade their normalized propositions.

    Supported annotations are ``[assert:fact=value]``,
    ``[unsupported:negative-fact-id]`` and ``[input:value]``. Duplicate fact
    declarations or malformed annotation prefixes invalidate the evidence.
    """
    book = fixture or load_book_fixture()
    assertion_rows = _ASSERTION.findall(response)
    assertions: dict[str, str] = {}
    duplicate = False
    for raw_fact, raw_value in assertion_rows:
        fact = raw_fact.casefold()
        value = _normalized(raw_value)
        duplicate = duplicate or fact in assertions
        assertions[fact] = value

    unsupported = tuple(row.casefold() for row in _UNSUPPORTED.findall(response))
    inputs = tuple(_normalized(row) for row in _INPUT.findall(response))
    malformed = (
        response.casefold().count("[assert:") != len(assertion_rows)
        or response.casefold().count("[unsupported:") != len(unsupported)
        or response.casefold().count("[input:") != len(inputs)
    )
    truth = _truth(book)
    invented = sum(
        value not in truth.get(fact, frozenset())
        for fact, value in assertions.items()
    )
    valid_negative_ids = {row.id for row in book.negative_facts}
    invented += sum(row not in valid_negative_ids for row in unsupported)
    return MappingProxyType(
        {
            "book_response_evidence_valid": not duplicate and not malformed,
            "response_claims": MappingProxyType(assertions),
            "current_assertions": MappingProxyType(assertions),
            "unsupported_fact_ids": unsupported,
            "unsupported_claim_count": invented,
            "derivation_inputs_used": inputs,
        }
    )