1"""Checks promptfooconfig.yaml and evaluates its assertions on recorded outputs, the way promptfoo would with the
2echo provider. It implements the assertion types listed in the lesson, as promptfoo documents them. Do not edit."""
3import json
4import re
5
6import yaml
7
8SUPPORTED = {"equals", "contains", "icontains", "not-contains", "not-icontains", "regex", "not-regex", "starts-with",
9"is-json", "contains-any", "icontains-any", "not-icontains-any"}
10TRUE = {"true", "1", "yes"}
11
12
13def render(value, variables):
14if isinstance(value, list):
15return [render(v, variables) for v in value]
16return re.sub(r"\{\{\s*(\w+)\s*\}\}", lambda m: str(variables.get(m.group(1), "")), str(value))
17
18
19def holds(a, output, variables):
20kind = a.get("type")
21value = render(a.get("value", ""), variables)
22if kind == "equals":
23return output == value
24if kind == "contains":
25return value in output
26if kind == "icontains":
27return value.lower() in output.lower()
28if kind == "not-contains":
29return value not in output
30if kind == "not-icontains":
31return value.lower() not in output.lower()
32if kind == "regex":
33return re.search(value, output) is not None
34if kind == "not-regex":
35return re.search(value, output) is None
36if kind == "starts-with":
37return output.startswith(value)
38if kind == "is-json":
39try:
40json.loads(output)
41return True
42except ValueError:
43return False
44values = value if isinstance(value, list) else [value]
45if kind == "contains-any":
46return any(v in output for v in values)
47if kind == "icontains-any":
48return any(v.lower() in output.lower() for v in values)
49if kind == "not-icontains-any":
50return not any(v.lower() in output.lower() for v in values)
51return False
52
53
54def main():
55facts = {"valid": False}
56try:
57cfg = yaml.safe_load(open("promptfooconfig.yaml"))
58except Exception as e: # noqa: BLE001
59print(json.dumps({**facts, "error": f"not valid YAML: {e}"}))
60return
61if not isinstance(cfg, dict):
62print(json.dumps({**facts, "error": "the config must be a YAML mapping"}))
63return
64env = {k: str(v).lower() for k, v in (cfg.get("env") or {}).items()}
65providers = cfg.get("providers") or []
66providers = [p if isinstance(p, str) else (p or {}).get("id", "") for p in (providers if isinstance(providers, list) else [providers])]
67prompts = cfg.get("prompts") or []
68prompts = prompts if isinstance(prompts, list) else [prompts]
69tests = cfg.get("tests")
70if tests == "file://recorded_tests.yaml":
71tests = yaml.safe_load(open("recorded_tests.yaml"))
72if not isinstance(tests, list):
73print(json.dumps({**facts, "error": "tests must be a list, or file://recorded_tests.yaml"}))
74return
75default = (cfg.get("defaultTest") or {}).get("assert") or []
76failed, unsupported, counts = [], set(), []
77for i, t in enumerate(tests):
78variables = t.get("vars") or {}
79asserts = list(default) + list(t.get("assert") or [])
80counts.append(len(asserts))
81output = render(prompts[0], variables) if prompts else ""
82for a in asserts:
83if a.get("type") not in SUPPORTED:
84unsupported.add(str(a.get("type")))
85continue
86if not holds(a, output, variables):
87failed.append(t.get("description") or f"test {i + 1}")
88break
89for t in sorted(set(failed)):
90print(f"FAIL {t}")
91print(json.dumps({
92"valid": True,
93"echo": providers == ["echo"],
94"prompt_is_output": prompts == ["{{output}}"],
95"telemetry_off": env.get("PROMPTFOO_DISABLE_TELEMETRY", "") in TRUE,
96"remote_off": env.get("PROMPTFOO_DISABLE_REMOTE_GENERATION", "") in TRUE,
97"tests": len(tests),
98"min_asserts": min(counts) if counts else 0,
99"unsupported": sorted(unsupported),
100"failed": sorted(set(failed)),
101}))
102
103
104main()
105
