From 2a2e0b41c2f92533a19745b119f76db3bf41bd4e Mon Sep 17 00:00:00 2001 From: Drew Ritter Date: Thu, 6 Aug 2026 14:49:13 -0700 Subject: [PATCH] fix(e2e): reject fenced and malformed scenario tables Filter backtick and tilde fenced examples before locating the E2E scenario-card table, so illustrative Markdown cannot satisfy the mechanical gate or shadow a later real table.\n\nParse only the documented leading/trailing-pipe row form and require a same-width delimiter row whose cells contain at least three hyphens with optional alignment colons. Missing or malformed delimiters now remain check failures instead of being silently treated as data.\n\nThe process-level harness records RED reproductions for both false-pass forms, fenced-only tables, and fenced examples before real tables, then verifies the focused GREEN behavior and prior contracts. --- .../scripts/check-cards-against-spec | 76 ++++++++++++++++--- .../test-check-cards-against-spec.sh | 52 +++++++++++++ 2 files changed, 118 insertions(+), 10 deletions(-) diff --git a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec index 4b491564..dc6281da 100755 --- a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec +++ b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec @@ -34,6 +34,35 @@ warn() { echo "warn: $1"; } # design spec: markdown re-wrapping must not defeat the verbatim check.) normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; } +# Exclude fenced examples from structural matching. This intentionally models +# only backtick/tilde fences; it is not a general Markdown parser. +without_fenced_code() { + awk ' + function fence_family(line, first, count) { + sub(/^[[:space:]]*/, "", line) + first = substr(line, 1, 1) + if (first != "`" && first != "~") return "" + count = 0 + while (substr(line, count + 1, 1) == first) count++ + return count >= 3 ? first : "" + } + { + marker = fence_family($0) + if (!in_fence && marker != "") { + in_fence = 1 + family = marker + next + } + if (in_fence && marker == family) { + in_fence = 0 + family = "" + next + } + if (!in_fence) print + } + ' "$1" +} + # Text of the card's Expected section only (case-insensitive heading match, # any ##+ level; section ends at the next heading or EOF). expected_section() { @@ -56,7 +85,7 @@ TABLE="$(awk ' } insec && /^[[:space:]]*\|/ { intable = 1; print; next } insec && intable { exit } -' "$SPEC")" +' < <(without_fenced_code "$SPEC"))" if [ -z "$TABLE" ]; then echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2 @@ -68,13 +97,24 @@ fi US=$'\x1f' CARD_COL=-1; FALS_COL=-1; ROWS=0 declare -a ROW_CARD ROW_FALS +HEADER_COLS=0; TABLE_VALID=1 lineno=0 while IFS= read -r line; do lineno=$((lineno + 1)) - esc="${line//\\|/$US}" + row="$(printf '%s' "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + case "$row" in + \|*\|) ;; + *) + fail "row $lineno: canonical table rows require leading and trailing pipes" + TABLE_VALID=0 + break + ;; + esac + row="${row#|}" + row="${row%|}" + esc="${row//\\|/$US}" IFS='|' read -r -a cells <<< "$esc" - # drop leading/trailing empty fields produced by the outer pipes trimmed=() for c in "${cells[@]}"; do c="${c//$US/|}" @@ -88,14 +128,28 @@ while IFS= read -r line; do [ "$low" = "card" ] && CARD_COL=$i [ "$low" = "falsification" ] && FALS_COL=$i done + HEADER_COLS=${#trimmed[@]} + if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then + fail "table header must name Card and Falsification columns" + TABLE_VALID=0 + break + fi continue fi - # separator row: cells of dashes/colons only - joined="$(printf '%s' "${trimmed[*]}" | tr -d ' :-')" - [ -z "$joined" ] && continue - if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then - fail "table header must name Card and Falsification columns" - break + if [ "$lineno" -eq 2 ]; then + delimiter_ok=1 + [ "${#trimmed[@]}" -eq "$HEADER_COLS" ] || delimiter_ok=0 + for c in "${trimmed[@]}"; do + if ! printf '%s\n' "$c" | grep -Eq '^:?-{3,}:?$'; then + delimiter_ok=0 + fi + done + if [ "$delimiter_ok" -ne 1 ]; then + fail "row 2: malformed table delimiter" + TABLE_VALID=0 + break + fi + continue fi card="${trimmed[$CARD_COL]:-}" falsif="${trimmed[$FALS_COL]:-}" @@ -107,7 +161,9 @@ while IFS= read -r line; do ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1)) done <<< "$TABLE" -[ "$ROWS" -ge 1 ] || fail "scenario table has no data rows" +if [ "$TABLE_VALID" -eq 1 ] && [ "$ROWS" -lt 1 ]; then + fail "scenario table has no data rows" +fi # --- checks 2-4 per row ----------------------------------------------------- i=0 diff --git a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh index 44385ef6..4dbe103a 100755 --- a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh +++ b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh @@ -202,6 +202,58 @@ assert_exit 0 "extra card -> exit 0" \ "$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards" assert_out_contains "extra-exploration" "warning names the extra card" +echo "scenario table requires a delimiter row" +make_spec "$TEST_ROOT/t10"; make_cards "$TEST_ROOT/t10/cards" +sed -i.bak '/^| --- | --- | --- |$/d' "$TEST_ROOT/t10/spec.md" +assert_exit 1 "header followed by data without delimiter -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t10/spec.md" "$TEST_ROOT/t10/cards" + +echo "scenario table requires a valid delimiter row" +make_spec "$TEST_ROOT/t11"; make_cards "$TEST_ROOT/t11/cards" +sed -i.bak 's/^| --- | --- | --- |$/| -- | --- | --- |/' "$TEST_ROOT/t11/spec.md" +assert_exit 1 "delimiter cells require at least three hyphens -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t11/spec.md" "$TEST_ROOT/t11/cards" + +echo "scenario table inside a fenced example does not count" +mkdir -p "$TEST_ROOT/t12/cards" +make_cards "$TEST_ROOT/t12/cards" +cat > "$TEST_ROOT/t12/spec.md" <<'EOF' +# Widget Design + +## E2E scenario cards + +```markdown +| Card | Covers | Falsification | +| --- | --- | --- | +| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. | +| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. | +``` +EOF +assert_exit 2 "fenced-only table -> exit 2" \ + "$CHECKER" "$TEST_ROOT/t12/spec.md" "$TEST_ROOT/t12/cards" + +echo "fenced example before a real table is ignored" +mkdir -p "$TEST_ROOT/t13/cards" +make_cards "$TEST_ROOT/t13/cards" +cat > "$TEST_ROOT/t13/spec.md" <<'EOF' +# Widget Design + +## E2E scenario cards + +~~~markdown +| Card | Covers | Falsification | +| --- | --- | --- | +| fake-card | Example only | If the example is absent, the scenario FAILS. | +~~~ + +| Card | Covers | Falsification | +| --- | --- | --- | +| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. | +| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. | +EOF +assert_exit 0 "real table after fenced example -> exit 0" \ + "$CHECKER" "$TEST_ROOT/t13/spec.md" "$TEST_ROOT/t13/cards" + echo "no scenario table" mkdir -p "$TEST_ROOT/t7/cards" printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md"