Braintrust

1
import braintrust
2
from respan import Respan
3
from respan_instrumentation_braintrust import BraintrustInstrumentor
4
5
WORKFLOW_NAME = "Braintrust Support Evaluation"
6
7
respan = Respan(
8
    app_name="braintrust-support-eval",
9
    instrumentations=[BraintrustInstrumentor()],
10
)
11
12
logger = braintrust.init_logger(
13
    project="Support Evaluation",
14
    async_flush=False,
15
)
16
17
try:
18
    with respan.propagate_attributes(
19
        group_identifier=WORKFLOW_NAME,
20
        custom_identifier="support-eval-001",
21
        metadata={"workflow_name": WORKFLOW_NAME, "framework": "braintrust"},
22
    ):
23
        with logger.start_span(name=f"{WORKFLOW_NAME}.workflow", type="eval") as root:
24
            with root.start_span(name="draft_answer", type="llm") as span:
25
                span.log(
26
                    input=[{
27
                        "role": "user",
28
                        "content": "Can I get a refund after 21 days?",
29
                    }],
30
                    output="Yes. Refunds are available within 30 days with a receipt.",
31
                    metadata={"model": "gpt-4o-mini"},
32
                    metrics={"prompt_tokens": 15, "completion_tokens": 16},
33
                    scores={"helpfulness": 0.95},
34
                )
35
36
            root.log(
37
                input={"case_id": "case-1024"},
38
                output={"decision": "eligible"},
39
                metadata={"workflow_name": WORKFLOW_NAME},
40
            )
41
42
        logger.flush()
43
finally:
44
    respan.flush()
45
    respan.shutdown()